1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include "opt_rss.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bus.h> 38 #include <sys/endian.h> 39 #include <sys/kernel.h> 40 #include <sys/kthread.h> 41 #include <sys/malloc.h> 42 #include <sys/mbuf.h> 43 #include <sys/module.h> 44 #include <sys/rman.h> 45 #include <sys/smp.h> 46 #include <sys/socket.h> 47 #include <sys/sockio.h> 48 #include <sys/sysctl.h> 49 #include <sys/taskqueue.h> 50 #include <sys/time.h> 51 #include <sys/eventhandler.h> 52 53 #include <machine/bus.h> 54 #include <machine/resource.h> 55 #include <machine/in_cksum.h> 56 57 #include <net/bpf.h> 58 #include <net/ethernet.h> 59 #include <net/if.h> 60 #include <net/if_var.h> 61 #include <net/if_arp.h> 62 #include <net/if_dl.h> 63 #include <net/if_media.h> 64 #include <net/if_types.h> 65 #include <net/if_vlan_var.h> 66 67 #include <netinet/in_systm.h> 68 #include <netinet/in.h> 69 #include <netinet/if_ether.h> 70 #include <netinet/ip.h> 71 #include <netinet/ip6.h> 72 #include <netinet/tcp.h> 73 #include <netinet/udp.h> 74 75 #include <dev/pci/pcivar.h> 76 #include <dev/pci/pcireg.h> 77 78 #include <vm/vm.h> 79 #include <vm/pmap.h> 80 81 #include "ena_datapath.h" 82 #include "ena.h" 83 #include "ena_sysctl.h" 84 #include "ena_rss.h" 85 86 #ifdef DEV_NETMAP 87 #include "ena_netmap.h" 88 #endif /* DEV_NETMAP */ 89 90 /********************************************************* 91 * Function prototypes 92 *********************************************************/ 93 static int ena_probe(device_t); 94 static void ena_intr_msix_mgmnt(void *); 95 static void ena_free_pci_resources(struct ena_adapter *); 96 static int ena_change_mtu(if_t, int); 97 static inline void ena_alloc_counters(counter_u64_t *, int); 98 static inline void ena_free_counters(counter_u64_t *, int); 99 static inline void ena_reset_counters(counter_u64_t *, int); 100 static void ena_init_io_rings_common(struct ena_adapter *, 101 struct ena_ring *, uint16_t); 102 static void ena_init_io_rings_basic(struct ena_adapter *); 103 static void ena_init_io_rings_advanced(struct ena_adapter *); 104 static void ena_init_io_rings(struct ena_adapter *); 105 static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int); 106 static void ena_free_all_io_rings_resources(struct ena_adapter *); 107 static int ena_setup_tx_dma_tag(struct ena_adapter *); 108 static int ena_free_tx_dma_tag(struct ena_adapter *); 109 static int ena_setup_rx_dma_tag(struct ena_adapter *); 110 static int ena_free_rx_dma_tag(struct ena_adapter *); 111 static void ena_release_all_tx_dmamap(struct ena_ring *); 112 static int ena_setup_tx_resources(struct ena_adapter *, int); 113 static void ena_free_tx_resources(struct ena_adapter *, int); 114 static int ena_setup_all_tx_resources(struct ena_adapter *); 115 static void ena_free_all_tx_resources(struct ena_adapter *); 116 static int ena_setup_rx_resources(struct ena_adapter *, unsigned int); 117 static void ena_free_rx_resources(struct ena_adapter *, unsigned int); 118 static int ena_setup_all_rx_resources(struct ena_adapter *); 119 static void ena_free_all_rx_resources(struct ena_adapter *); 120 static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *, 121 struct ena_rx_buffer *); 122 static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *, 123 struct ena_rx_buffer *); 124 static void ena_free_rx_bufs(struct ena_adapter *, unsigned int); 125 static void ena_refill_all_rx_bufs(struct ena_adapter *); 126 static void ena_free_all_rx_bufs(struct ena_adapter *); 127 static void ena_free_tx_bufs(struct ena_adapter *, unsigned int); 128 static void ena_free_all_tx_bufs(struct ena_adapter *); 129 static void ena_destroy_all_tx_queues(struct ena_adapter *); 130 static void ena_destroy_all_rx_queues(struct ena_adapter *); 131 static void ena_destroy_all_io_queues(struct ena_adapter *); 132 static int ena_create_io_queues(struct ena_adapter *); 133 static int ena_handle_msix(void *); 134 static int ena_enable_msix(struct ena_adapter *); 135 static void ena_setup_mgmnt_intr(struct ena_adapter *); 136 static int ena_setup_io_intr(struct ena_adapter *); 137 static int ena_request_mgmnt_irq(struct ena_adapter *); 138 static int ena_request_io_irq(struct ena_adapter *); 139 static void ena_free_mgmnt_irq(struct ena_adapter *); 140 static void ena_free_io_irq(struct ena_adapter *); 141 static void ena_free_irqs(struct ena_adapter*); 142 static void ena_disable_msix(struct ena_adapter *); 143 static void ena_unmask_all_io_irqs(struct ena_adapter *); 144 static int ena_up_complete(struct ena_adapter *); 145 static uint64_t ena_get_counter(if_t, ift_counter); 146 static int ena_media_change(if_t); 147 static void ena_media_status(if_t, struct ifmediareq *); 148 static void ena_init(void *); 149 static int ena_ioctl(if_t, u_long, caddr_t); 150 static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); 151 static void ena_update_host_info(struct ena_admin_host_info *, if_t); 152 static void ena_update_hwassist(struct ena_adapter *); 153 static int ena_setup_ifnet(device_t, struct ena_adapter *, 154 struct ena_com_dev_get_features_ctx *); 155 static int ena_enable_wc(device_t, struct resource *); 156 static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *, 157 struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *); 158 static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev *, 159 struct ena_com_dev_get_features_ctx *); 160 static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *); 161 static void ena_config_host_info(struct ena_com_dev *, device_t); 162 static int ena_attach(device_t); 163 static int ena_detach(device_t); 164 static int ena_device_init(struct ena_adapter *, device_t, 165 struct ena_com_dev_get_features_ctx *, int *); 166 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *); 167 static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *); 168 static void unimplemented_aenq_handler(void *, 169 struct ena_admin_aenq_entry *); 170 static int ena_copy_eni_metrics(struct ena_adapter *); 171 static void ena_timer_service(void *); 172 173 static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION; 174 175 static ena_vendor_info_t ena_vendor_info_array[] = { 176 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0}, 177 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF_RSERV0, 0}, 178 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0}, 179 { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF_RSERV0, 0}, 180 /* Last entry */ 181 { 0, 0, 0 } 182 }; 183 184 struct sx ena_global_lock; 185 186 /* 187 * Contains pointers to event handlers, e.g. link state chage. 188 */ 189 static struct ena_aenq_handlers aenq_handlers; 190 191 void 192 ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) 193 { 194 if (error != 0) 195 return; 196 *(bus_addr_t *) arg = segs[0].ds_addr; 197 } 198 199 int 200 ena_dma_alloc(device_t dmadev, bus_size_t size, 201 ena_mem_handle_t *dma, int mapflags, bus_size_t alignment, int domain) 202 { 203 struct ena_adapter* adapter = device_get_softc(dmadev); 204 device_t pdev = adapter->pdev; 205 uint32_t maxsize; 206 uint64_t dma_space_addr; 207 int error; 208 209 maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; 210 211 dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); 212 if (unlikely(dma_space_addr == 0)) 213 dma_space_addr = BUS_SPACE_MAXADDR; 214 215 error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */ 216 alignment, 0, /* alignment, bounds */ 217 dma_space_addr, /* lowaddr of exclusion window */ 218 BUS_SPACE_MAXADDR,/* highaddr of exclusion window */ 219 NULL, NULL, /* filter, filterarg */ 220 maxsize, /* maxsize */ 221 1, /* nsegments */ 222 maxsize, /* maxsegsize */ 223 BUS_DMA_ALLOCNOW, /* flags */ 224 NULL, /* lockfunc */ 225 NULL, /* lockarg */ 226 &dma->tag); 227 if (unlikely(error != 0)) { 228 ena_log(pdev, ERR, "bus_dma_tag_create failed: %d\n", error); 229 goto fail_tag; 230 } 231 232 error = bus_dma_tag_set_domain(dma->tag, domain); 233 if (unlikely(error != 0)) { 234 ena_log(pdev, ERR, "bus_dma_tag_set_domain failed: %d\n", 235 error); 236 goto fail_map_create; 237 } 238 239 error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr, 240 BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map); 241 if (unlikely(error != 0)) { 242 ena_log(pdev, ERR, "bus_dmamem_alloc(%ju) failed: %d\n", 243 (uintmax_t)size, error); 244 goto fail_map_create; 245 } 246 247 dma->paddr = 0; 248 error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, 249 size, ena_dmamap_callback, &dma->paddr, mapflags); 250 if (unlikely((error != 0) || (dma->paddr == 0))) { 251 ena_log(pdev, ERR, "bus_dmamap_load failed: %d\n", error); 252 goto fail_map_load; 253 } 254 255 bus_dmamap_sync(dma->tag, dma->map, 256 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 257 258 return (0); 259 260 fail_map_load: 261 bus_dmamem_free(dma->tag, dma->vaddr, dma->map); 262 fail_map_create: 263 bus_dma_tag_destroy(dma->tag); 264 fail_tag: 265 dma->tag = NULL; 266 dma->vaddr = NULL; 267 dma->paddr = 0; 268 269 return (error); 270 } 271 272 static void 273 ena_free_pci_resources(struct ena_adapter *adapter) 274 { 275 device_t pdev = adapter->pdev; 276 277 if (adapter->memory != NULL) { 278 bus_release_resource(pdev, SYS_RES_MEMORY, 279 PCIR_BAR(ENA_MEM_BAR), adapter->memory); 280 } 281 282 if (adapter->registers != NULL) { 283 bus_release_resource(pdev, SYS_RES_MEMORY, 284 PCIR_BAR(ENA_REG_BAR), adapter->registers); 285 } 286 287 if (adapter->msix != NULL) { 288 bus_release_resource(pdev, SYS_RES_MEMORY, 289 adapter->msix_rid, adapter->msix); 290 } 291 } 292 293 static int 294 ena_probe(device_t dev) 295 { 296 ena_vendor_info_t *ent; 297 char adapter_name[60]; 298 uint16_t pci_vendor_id = 0; 299 uint16_t pci_device_id = 0; 300 301 pci_vendor_id = pci_get_vendor(dev); 302 pci_device_id = pci_get_device(dev); 303 304 ent = ena_vendor_info_array; 305 while (ent->vendor_id != 0) { 306 if ((pci_vendor_id == ent->vendor_id) && 307 (pci_device_id == ent->device_id)) { 308 ena_log_raw(DBG, "vendor=%x device=%x\n", 309 pci_vendor_id, pci_device_id); 310 311 sprintf(adapter_name, DEVICE_DESC); 312 device_set_desc_copy(dev, adapter_name); 313 return (BUS_PROBE_DEFAULT); 314 } 315 316 ent++; 317 318 } 319 320 return (ENXIO); 321 } 322 323 static int 324 ena_change_mtu(if_t ifp, int new_mtu) 325 { 326 struct ena_adapter *adapter = if_getsoftc(ifp); 327 device_t pdev = adapter->pdev; 328 int rc; 329 330 if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { 331 ena_log(pdev, ERR, "Invalid MTU setting. " 332 "new_mtu: %d max mtu: %d min mtu: %d\n", 333 new_mtu, adapter->max_mtu, ENA_MIN_MTU); 334 return (EINVAL); 335 } 336 337 rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); 338 if (likely(rc == 0)) { 339 ena_log(pdev, DBG, "set MTU to %d\n", new_mtu); 340 if_setmtu(ifp, new_mtu); 341 } else { 342 ena_log(pdev, ERR, "Failed to set MTU to %d\n", new_mtu); 343 } 344 345 return (rc); 346 } 347 348 static inline void 349 ena_alloc_counters(counter_u64_t *begin, int size) 350 { 351 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 352 353 for (; begin < end; ++begin) 354 *begin = counter_u64_alloc(M_WAITOK); 355 } 356 357 static inline void 358 ena_free_counters(counter_u64_t *begin, int size) 359 { 360 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 361 362 for (; begin < end; ++begin) 363 counter_u64_free(*begin); 364 } 365 366 static inline void 367 ena_reset_counters(counter_u64_t *begin, int size) 368 { 369 counter_u64_t *end = (counter_u64_t *)((char *)begin + size); 370 371 for (; begin < end; ++begin) 372 counter_u64_zero(*begin); 373 } 374 375 static void 376 ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, 377 uint16_t qid) 378 { 379 380 ring->qid = qid; 381 ring->adapter = adapter; 382 ring->ena_dev = adapter->ena_dev; 383 ring->first_interrupt = false; 384 ring->no_interrupt_event_cnt = 0; 385 } 386 387 static void 388 ena_init_io_rings_basic(struct ena_adapter *adapter) 389 { 390 struct ena_com_dev *ena_dev; 391 struct ena_ring *txr, *rxr; 392 struct ena_que *que; 393 int i; 394 395 ena_dev = adapter->ena_dev; 396 397 for (i = 0; i < adapter->num_io_queues; i++) { 398 txr = &adapter->tx_ring[i]; 399 rxr = &adapter->rx_ring[i]; 400 401 /* TX/RX common ring state */ 402 ena_init_io_rings_common(adapter, txr, i); 403 ena_init_io_rings_common(adapter, rxr, i); 404 405 /* TX specific ring state */ 406 txr->tx_max_header_size = ena_dev->tx_max_header_size; 407 txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; 408 409 que = &adapter->que[i]; 410 que->adapter = adapter; 411 que->id = i; 412 que->tx_ring = txr; 413 que->rx_ring = rxr; 414 415 txr->que = que; 416 rxr->que = que; 417 418 rxr->empty_rx_queue = 0; 419 rxr->rx_mbuf_sz = ena_mbuf_sz; 420 } 421 } 422 423 static void 424 ena_init_io_rings_advanced(struct ena_adapter *adapter) 425 { 426 struct ena_ring *txr, *rxr; 427 int i; 428 429 for (i = 0; i < adapter->num_io_queues; i++) { 430 txr = &adapter->tx_ring[i]; 431 rxr = &adapter->rx_ring[i]; 432 433 /* Allocate a buf ring */ 434 txr->buf_ring_size = adapter->buf_ring_size; 435 txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, 436 M_WAITOK, &txr->ring_mtx); 437 438 /* Allocate Tx statistics. */ 439 ena_alloc_counters((counter_u64_t *)&txr->tx_stats, 440 sizeof(txr->tx_stats)); 441 442 /* Allocate Rx statistics. */ 443 ena_alloc_counters((counter_u64_t *)&rxr->rx_stats, 444 sizeof(rxr->rx_stats)); 445 446 /* Initialize locks */ 447 snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)", 448 device_get_nameunit(adapter->pdev), i); 449 snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)", 450 device_get_nameunit(adapter->pdev), i); 451 452 mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF); 453 } 454 } 455 456 static void 457 ena_init_io_rings(struct ena_adapter *adapter) 458 { 459 /* 460 * IO rings initialization can be divided into the 2 steps: 461 * 1. Initialize variables and fields with initial values and copy 462 * them from adapter/ena_dev (basic) 463 * 2. Allocate mutex, counters and buf_ring (advanced) 464 */ 465 ena_init_io_rings_basic(adapter); 466 ena_init_io_rings_advanced(adapter); 467 } 468 469 static void 470 ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) 471 { 472 struct ena_ring *txr = &adapter->tx_ring[qid]; 473 struct ena_ring *rxr = &adapter->rx_ring[qid]; 474 475 ena_free_counters((counter_u64_t *)&txr->tx_stats, 476 sizeof(txr->tx_stats)); 477 ena_free_counters((counter_u64_t *)&rxr->rx_stats, 478 sizeof(rxr->rx_stats)); 479 480 ENA_RING_MTX_LOCK(txr); 481 drbr_free(txr->br, M_DEVBUF); 482 ENA_RING_MTX_UNLOCK(txr); 483 484 mtx_destroy(&txr->ring_mtx); 485 } 486 487 static void 488 ena_free_all_io_rings_resources(struct ena_adapter *adapter) 489 { 490 int i; 491 492 for (i = 0; i < adapter->num_io_queues; i++) 493 ena_free_io_ring_resources(adapter, i); 494 495 } 496 497 static int 498 ena_setup_tx_dma_tag(struct ena_adapter *adapter) 499 { 500 int ret; 501 502 /* Create DMA tag for Tx buffers */ 503 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), 504 1, 0, /* alignment, bounds */ 505 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 506 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 507 NULL, NULL, /* filter, filterarg */ 508 ENA_TSO_MAXSIZE, /* maxsize */ 509 adapter->max_tx_sgl_size - 1, /* nsegments */ 510 ENA_TSO_MAXSIZE, /* maxsegsize */ 511 0, /* flags */ 512 NULL, /* lockfunc */ 513 NULL, /* lockfuncarg */ 514 &adapter->tx_buf_tag); 515 516 return (ret); 517 } 518 519 static int 520 ena_free_tx_dma_tag(struct ena_adapter *adapter) 521 { 522 int ret; 523 524 ret = bus_dma_tag_destroy(adapter->tx_buf_tag); 525 526 if (likely(ret == 0)) 527 adapter->tx_buf_tag = NULL; 528 529 return (ret); 530 } 531 532 static int 533 ena_setup_rx_dma_tag(struct ena_adapter *adapter) 534 { 535 int ret; 536 537 /* Create DMA tag for Rx buffers*/ 538 ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ 539 1, 0, /* alignment, bounds */ 540 ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ 541 BUS_SPACE_MAXADDR, /* highaddr of excl window */ 542 NULL, NULL, /* filter, filterarg */ 543 ena_mbuf_sz, /* maxsize */ 544 adapter->max_rx_sgl_size, /* nsegments */ 545 ena_mbuf_sz, /* maxsegsize */ 546 0, /* flags */ 547 NULL, /* lockfunc */ 548 NULL, /* lockarg */ 549 &adapter->rx_buf_tag); 550 551 return (ret); 552 } 553 554 static int 555 ena_free_rx_dma_tag(struct ena_adapter *adapter) 556 { 557 int ret; 558 559 ret = bus_dma_tag_destroy(adapter->rx_buf_tag); 560 561 if (likely(ret == 0)) 562 adapter->rx_buf_tag = NULL; 563 564 return (ret); 565 } 566 567 static void 568 ena_release_all_tx_dmamap(struct ena_ring *tx_ring) 569 { 570 struct ena_adapter *adapter = tx_ring->adapter; 571 struct ena_tx_buffer *tx_info; 572 bus_dma_tag_t tx_tag = adapter->tx_buf_tag;; 573 int i; 574 #ifdef DEV_NETMAP 575 struct ena_netmap_tx_info *nm_info; 576 int j; 577 #endif /* DEV_NETMAP */ 578 579 for (i = 0; i < tx_ring->ring_size; ++i) { 580 tx_info = &tx_ring->tx_buffer_info[i]; 581 #ifdef DEV_NETMAP 582 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 583 nm_info = &tx_info->nm_info; 584 for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) { 585 if (nm_info->map_seg[j] != NULL) { 586 bus_dmamap_destroy(tx_tag, 587 nm_info->map_seg[j]); 588 nm_info->map_seg[j] = NULL; 589 } 590 } 591 } 592 #endif /* DEV_NETMAP */ 593 if (tx_info->dmamap != NULL) { 594 bus_dmamap_destroy(tx_tag, tx_info->dmamap); 595 tx_info->dmamap = NULL; 596 } 597 } 598 } 599 600 /** 601 * ena_setup_tx_resources - allocate Tx resources (Descriptors) 602 * @adapter: network interface device structure 603 * @qid: queue index 604 * 605 * Returns 0 on success, otherwise on failure. 606 **/ 607 static int 608 ena_setup_tx_resources(struct ena_adapter *adapter, int qid) 609 { 610 device_t pdev = adapter->pdev; 611 char thread_name[MAXCOMLEN + 1]; 612 struct ena_que *que = &adapter->que[qid]; 613 struct ena_ring *tx_ring = que->tx_ring; 614 cpuset_t *cpu_mask = NULL; 615 int size, i, err; 616 #ifdef DEV_NETMAP 617 bus_dmamap_t *map; 618 int j; 619 620 ena_netmap_reset_tx_ring(adapter, qid); 621 #endif /* DEV_NETMAP */ 622 623 size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; 624 625 tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 626 if (unlikely(tx_ring->tx_buffer_info == NULL)) 627 return (ENOMEM); 628 629 size = sizeof(uint16_t) * tx_ring->ring_size; 630 tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 631 if (unlikely(tx_ring->free_tx_ids == NULL)) 632 goto err_buf_info_free; 633 634 size = tx_ring->tx_max_header_size; 635 tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF, 636 M_NOWAIT | M_ZERO); 637 if (unlikely(tx_ring->push_buf_intermediate_buf == NULL)) 638 goto err_tx_ids_free; 639 640 /* Req id stack for TX OOO completions */ 641 for (i = 0; i < tx_ring->ring_size; i++) 642 tx_ring->free_tx_ids[i] = i; 643 644 /* Reset TX statistics. */ 645 ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats, 646 sizeof(tx_ring->tx_stats)); 647 648 tx_ring->next_to_use = 0; 649 tx_ring->next_to_clean = 0; 650 tx_ring->acum_pkts = 0; 651 652 /* Make sure that drbr is empty */ 653 ENA_RING_MTX_LOCK(tx_ring); 654 drbr_flush(adapter->ifp, tx_ring->br); 655 ENA_RING_MTX_UNLOCK(tx_ring); 656 657 /* ... and create the buffer DMA maps */ 658 for (i = 0; i < tx_ring->ring_size; i++) { 659 err = bus_dmamap_create(adapter->tx_buf_tag, 0, 660 &tx_ring->tx_buffer_info[i].dmamap); 661 if (unlikely(err != 0)) { 662 ena_log(pdev, ERR, 663 "Unable to create Tx DMA map for buffer %d\n", 664 i); 665 goto err_map_release; 666 } 667 668 #ifdef DEV_NETMAP 669 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 670 map = tx_ring->tx_buffer_info[i].nm_info.map_seg; 671 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 672 err = bus_dmamap_create(adapter->tx_buf_tag, 0, 673 &map[j]); 674 if (unlikely(err != 0)) { 675 ena_log(pdev, ERR, 676 "Unable to create " 677 "Tx DMA for buffer %d %d\n", i, j); 678 goto err_map_release; 679 } 680 } 681 } 682 #endif /* DEV_NETMAP */ 683 } 684 685 /* Allocate taskqueues */ 686 TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring); 687 tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT, 688 taskqueue_thread_enqueue, &tx_ring->enqueue_tq); 689 if (unlikely(tx_ring->enqueue_tq == NULL)) { 690 ena_log(pdev, ERR, 691 "Unable to create taskqueue for enqueue task\n"); 692 i = tx_ring->ring_size; 693 goto err_map_release; 694 } 695 696 tx_ring->running = true; 697 698 #ifdef RSS 699 cpu_mask = &que->cpu_mask; 700 snprintf(thread_name, sizeof(thread_name), "%s txeq %d", 701 device_get_nameunit(adapter->pdev), que->cpu); 702 #else 703 snprintf(thread_name, sizeof(thread_name), "%s txeq %d", 704 device_get_nameunit(adapter->pdev), que->id); 705 #endif 706 taskqueue_start_threads_cpuset(&tx_ring->enqueue_tq, 1, PI_NET, 707 cpu_mask, "%s", thread_name); 708 709 return (0); 710 711 err_map_release: 712 ena_release_all_tx_dmamap(tx_ring); 713 err_tx_ids_free: 714 free(tx_ring->free_tx_ids, M_DEVBUF); 715 tx_ring->free_tx_ids = NULL; 716 err_buf_info_free: 717 free(tx_ring->tx_buffer_info, M_DEVBUF); 718 tx_ring->tx_buffer_info = NULL; 719 720 return (ENOMEM); 721 } 722 723 /** 724 * ena_free_tx_resources - Free Tx Resources per Queue 725 * @adapter: network interface device structure 726 * @qid: queue index 727 * 728 * Free all transmit software resources 729 **/ 730 static void 731 ena_free_tx_resources(struct ena_adapter *adapter, int qid) 732 { 733 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 734 #ifdef DEV_NETMAP 735 struct ena_netmap_tx_info *nm_info; 736 int j; 737 #endif /* DEV_NETMAP */ 738 739 while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, 740 NULL)) 741 taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task); 742 743 taskqueue_free(tx_ring->enqueue_tq); 744 745 ENA_RING_MTX_LOCK(tx_ring); 746 /* Flush buffer ring, */ 747 drbr_flush(adapter->ifp, tx_ring->br); 748 749 /* Free buffer DMA maps, */ 750 for (int i = 0; i < tx_ring->ring_size; i++) { 751 bus_dmamap_sync(adapter->tx_buf_tag, 752 tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE); 753 bus_dmamap_unload(adapter->tx_buf_tag, 754 tx_ring->tx_buffer_info[i].dmamap); 755 bus_dmamap_destroy(adapter->tx_buf_tag, 756 tx_ring->tx_buffer_info[i].dmamap); 757 758 #ifdef DEV_NETMAP 759 if (adapter->ifp->if_capenable & IFCAP_NETMAP) { 760 nm_info = &tx_ring->tx_buffer_info[i].nm_info; 761 for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { 762 if (nm_info->socket_buf_idx[j] != 0) { 763 bus_dmamap_sync(adapter->tx_buf_tag, 764 nm_info->map_seg[j], 765 BUS_DMASYNC_POSTWRITE); 766 ena_netmap_unload(adapter, 767 nm_info->map_seg[j]); 768 } 769 bus_dmamap_destroy(adapter->tx_buf_tag, 770 nm_info->map_seg[j]); 771 nm_info->socket_buf_idx[j] = 0; 772 } 773 } 774 #endif /* DEV_NETMAP */ 775 776 m_freem(tx_ring->tx_buffer_info[i].mbuf); 777 tx_ring->tx_buffer_info[i].mbuf = NULL; 778 } 779 ENA_RING_MTX_UNLOCK(tx_ring); 780 781 /* And free allocated memory. */ 782 free(tx_ring->tx_buffer_info, M_DEVBUF); 783 tx_ring->tx_buffer_info = NULL; 784 785 free(tx_ring->free_tx_ids, M_DEVBUF); 786 tx_ring->free_tx_ids = NULL; 787 788 free(tx_ring->push_buf_intermediate_buf, M_DEVBUF); 789 tx_ring->push_buf_intermediate_buf = NULL; 790 } 791 792 /** 793 * ena_setup_all_tx_resources - allocate all queues Tx resources 794 * @adapter: network interface device structure 795 * 796 * Returns 0 on success, otherwise on failure. 797 **/ 798 static int 799 ena_setup_all_tx_resources(struct ena_adapter *adapter) 800 { 801 int i, rc; 802 803 for (i = 0; i < adapter->num_io_queues; i++) { 804 rc = ena_setup_tx_resources(adapter, i); 805 if (rc != 0) { 806 ena_log(adapter->pdev, ERR, 807 "Allocation for Tx Queue %u failed\n", i); 808 goto err_setup_tx; 809 } 810 } 811 812 return (0); 813 814 err_setup_tx: 815 /* Rewind the index freeing the rings as we go */ 816 while (i--) 817 ena_free_tx_resources(adapter, i); 818 return (rc); 819 } 820 821 /** 822 * ena_free_all_tx_resources - Free Tx Resources for All Queues 823 * @adapter: network interface device structure 824 * 825 * Free all transmit software resources 826 **/ 827 static void 828 ena_free_all_tx_resources(struct ena_adapter *adapter) 829 { 830 int i; 831 832 for (i = 0; i < adapter->num_io_queues; i++) 833 ena_free_tx_resources(adapter, i); 834 } 835 836 /** 837 * ena_setup_rx_resources - allocate Rx resources (Descriptors) 838 * @adapter: network interface device structure 839 * @qid: queue index 840 * 841 * Returns 0 on success, otherwise on failure. 842 **/ 843 static int 844 ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) 845 { 846 device_t pdev = adapter->pdev; 847 struct ena_que *que = &adapter->que[qid]; 848 struct ena_ring *rx_ring = que->rx_ring; 849 int size, err, i; 850 851 size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; 852 853 #ifdef DEV_NETMAP 854 ena_netmap_reset_rx_ring(adapter, qid); 855 rx_ring->initialized = false; 856 #endif /* DEV_NETMAP */ 857 858 /* 859 * Alloc extra element so in rx path 860 * we can always prefetch rx_info + 1 861 */ 862 size += sizeof(struct ena_rx_buffer); 863 864 rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO); 865 866 size = sizeof(uint16_t) * rx_ring->ring_size; 867 rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK); 868 869 for (i = 0; i < rx_ring->ring_size; i++) 870 rx_ring->free_rx_ids[i] = i; 871 872 /* Reset RX statistics. */ 873 ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats, 874 sizeof(rx_ring->rx_stats)); 875 876 rx_ring->next_to_clean = 0; 877 rx_ring->next_to_use = 0; 878 879 /* ... and create the buffer DMA maps */ 880 for (i = 0; i < rx_ring->ring_size; i++) { 881 err = bus_dmamap_create(adapter->rx_buf_tag, 0, 882 &(rx_ring->rx_buffer_info[i].map)); 883 if (err != 0) { 884 ena_log(pdev, ERR, 885 "Unable to create Rx DMA map for buffer %d\n", i); 886 goto err_buf_info_unmap; 887 } 888 } 889 890 /* Create LRO for the ring */ 891 if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) { 892 int err = tcp_lro_init(&rx_ring->lro); 893 if (err != 0) { 894 ena_log(pdev, ERR, "LRO[%d] Initialization failed!\n", 895 qid); 896 } else { 897 ena_log(pdev, DBG, "RX Soft LRO[%d] Initialized\n", 898 qid); 899 rx_ring->lro.ifp = adapter->ifp; 900 } 901 } 902 903 return (0); 904 905 err_buf_info_unmap: 906 while (i--) { 907 bus_dmamap_destroy(adapter->rx_buf_tag, 908 rx_ring->rx_buffer_info[i].map); 909 } 910 911 free(rx_ring->free_rx_ids, M_DEVBUF); 912 rx_ring->free_rx_ids = NULL; 913 free(rx_ring->rx_buffer_info, M_DEVBUF); 914 rx_ring->rx_buffer_info = NULL; 915 return (ENOMEM); 916 } 917 918 /** 919 * ena_free_rx_resources - Free Rx Resources 920 * @adapter: network interface device structure 921 * @qid: queue index 922 * 923 * Free all receive software resources 924 **/ 925 static void 926 ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid) 927 { 928 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 929 930 /* Free buffer DMA maps, */ 931 for (int i = 0; i < rx_ring->ring_size; i++) { 932 bus_dmamap_sync(adapter->rx_buf_tag, 933 rx_ring->rx_buffer_info[i].map, BUS_DMASYNC_POSTREAD); 934 m_freem(rx_ring->rx_buffer_info[i].mbuf); 935 rx_ring->rx_buffer_info[i].mbuf = NULL; 936 bus_dmamap_unload(adapter->rx_buf_tag, 937 rx_ring->rx_buffer_info[i].map); 938 bus_dmamap_destroy(adapter->rx_buf_tag, 939 rx_ring->rx_buffer_info[i].map); 940 } 941 942 /* free LRO resources, */ 943 tcp_lro_free(&rx_ring->lro); 944 945 /* free allocated memory */ 946 free(rx_ring->rx_buffer_info, M_DEVBUF); 947 rx_ring->rx_buffer_info = NULL; 948 949 free(rx_ring->free_rx_ids, M_DEVBUF); 950 rx_ring->free_rx_ids = NULL; 951 } 952 953 /** 954 * ena_setup_all_rx_resources - allocate all queues Rx resources 955 * @adapter: network interface device structure 956 * 957 * Returns 0 on success, otherwise on failure. 958 **/ 959 static int 960 ena_setup_all_rx_resources(struct ena_adapter *adapter) 961 { 962 int i, rc = 0; 963 964 for (i = 0; i < adapter->num_io_queues; i++) { 965 rc = ena_setup_rx_resources(adapter, i); 966 if (rc != 0) { 967 ena_log(adapter->pdev, ERR, 968 "Allocation for Rx Queue %u failed\n", i); 969 goto err_setup_rx; 970 } 971 } 972 return (0); 973 974 err_setup_rx: 975 /* rewind the index freeing the rings as we go */ 976 while (i--) 977 ena_free_rx_resources(adapter, i); 978 return (rc); 979 } 980 981 /** 982 * ena_free_all_rx_resources - Free Rx resources for all queues 983 * @adapter: network interface device structure 984 * 985 * Free all receive software resources 986 **/ 987 static void 988 ena_free_all_rx_resources(struct ena_adapter *adapter) 989 { 990 int i; 991 992 for (i = 0; i < adapter->num_io_queues; i++) 993 ena_free_rx_resources(adapter, i); 994 } 995 996 static inline int 997 ena_alloc_rx_mbuf(struct ena_adapter *adapter, 998 struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info) 999 { 1000 device_t pdev = adapter->pdev; 1001 struct ena_com_buf *ena_buf; 1002 bus_dma_segment_t segs[1]; 1003 int nsegs, error; 1004 int mlen; 1005 1006 /* if previous allocated frag is not used */ 1007 if (unlikely(rx_info->mbuf != NULL)) 1008 return (0); 1009 1010 /* Get mbuf using UMA allocator */ 1011 rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, 1012 rx_ring->rx_mbuf_sz); 1013 1014 if (unlikely(rx_info->mbuf == NULL)) { 1015 counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1); 1016 rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1017 if (unlikely(rx_info->mbuf == NULL)) { 1018 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1); 1019 return (ENOMEM); 1020 } 1021 mlen = MCLBYTES; 1022 } else { 1023 mlen = rx_ring->rx_mbuf_sz; 1024 } 1025 /* Set mbuf length*/ 1026 rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen; 1027 1028 /* Map packets for DMA */ 1029 ena_log(pdev, DBG, "Using tag %p for buffers' DMA mapping, mbuf %p len: %d\n", 1030 adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len); 1031 error = bus_dmamap_load_mbuf_sg(adapter->rx_buf_tag, rx_info->map, 1032 rx_info->mbuf, segs, &nsegs, BUS_DMA_NOWAIT); 1033 if (unlikely((error != 0) || (nsegs != 1))) { 1034 ena_log(pdev, WARN, 1035 "failed to map mbuf, error: %d, nsegs: %d\n", error, nsegs); 1036 counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1); 1037 goto exit; 1038 1039 } 1040 1041 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD); 1042 1043 ena_buf = &rx_info->ena_buf; 1044 ena_buf->paddr = segs[0].ds_addr; 1045 ena_buf->len = mlen; 1046 1047 ena_log(pdev, DBG, "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n", 1048 rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr); 1049 1050 return (0); 1051 1052 exit: 1053 m_freem(rx_info->mbuf); 1054 rx_info->mbuf = NULL; 1055 return (EFAULT); 1056 } 1057 1058 static void 1059 ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring, 1060 struct ena_rx_buffer *rx_info) 1061 { 1062 1063 if (rx_info->mbuf == NULL) { 1064 ena_log(adapter->pdev, WARN, 1065 "Trying to free unallocated buffer\n"); 1066 return; 1067 } 1068 1069 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, 1070 BUS_DMASYNC_POSTREAD); 1071 bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map); 1072 m_freem(rx_info->mbuf); 1073 rx_info->mbuf = NULL; 1074 } 1075 1076 /** 1077 * ena_refill_rx_bufs - Refills ring with descriptors 1078 * @rx_ring: the ring which we want to feed with free descriptors 1079 * @num: number of descriptors to refill 1080 * Refills the ring with newly allocated DMA-mapped mbufs for receiving 1081 **/ 1082 int 1083 ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num) 1084 { 1085 struct ena_adapter *adapter = rx_ring->adapter; 1086 device_t pdev = adapter->pdev; 1087 uint16_t next_to_use, req_id; 1088 uint32_t i; 1089 int rc; 1090 1091 ena_log_io(adapter->pdev, DBG, "refill qid: %d\n", rx_ring->qid); 1092 1093 next_to_use = rx_ring->next_to_use; 1094 1095 for (i = 0; i < num; i++) { 1096 struct ena_rx_buffer *rx_info; 1097 1098 ena_log_io(pdev, DBG, "RX buffer - next to use: %d\n", 1099 next_to_use); 1100 1101 req_id = rx_ring->free_rx_ids[next_to_use]; 1102 rx_info = &rx_ring->rx_buffer_info[req_id]; 1103 #ifdef DEV_NETMAP 1104 if (ena_rx_ring_in_netmap(adapter, rx_ring->qid)) 1105 rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info); 1106 else 1107 #endif /* DEV_NETMAP */ 1108 rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info); 1109 if (unlikely(rc != 0)) { 1110 ena_log_io(pdev, WARN, 1111 "failed to alloc buffer for rx queue %d\n", 1112 rx_ring->qid); 1113 break; 1114 } 1115 rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq, 1116 &rx_info->ena_buf, req_id); 1117 if (unlikely(rc != 0)) { 1118 ena_log_io(pdev, WARN, 1119 "failed to add buffer for rx queue %d\n", 1120 rx_ring->qid); 1121 break; 1122 } 1123 next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use, 1124 rx_ring->ring_size); 1125 } 1126 1127 if (unlikely(i < num)) { 1128 counter_u64_add(rx_ring->rx_stats.refil_partial, 1); 1129 ena_log_io(pdev, WARN, 1130 "refilled rx qid %d with only %d mbufs (from %d)\n", 1131 rx_ring->qid, i, num); 1132 } 1133 1134 if (likely(i != 0)) 1135 ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq); 1136 1137 rx_ring->next_to_use = next_to_use; 1138 return (i); 1139 } 1140 1141 int 1142 ena_update_buf_ring_size(struct ena_adapter *adapter, 1143 uint32_t new_buf_ring_size) 1144 { 1145 uint32_t old_buf_ring_size; 1146 int rc = 0; 1147 bool dev_was_up; 1148 1149 old_buf_ring_size = adapter->buf_ring_size; 1150 adapter->buf_ring_size = new_buf_ring_size; 1151 1152 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1153 ena_down(adapter); 1154 1155 /* Reconfigure buf ring for all Tx rings. */ 1156 ena_free_all_io_rings_resources(adapter); 1157 ena_init_io_rings_advanced(adapter); 1158 if (dev_was_up) { 1159 /* 1160 * If ena_up() fails, it's not because of recent buf_ring size 1161 * changes. Because of that, we just want to revert old drbr 1162 * value and trigger the reset because something else had to 1163 * go wrong. 1164 */ 1165 rc = ena_up(adapter); 1166 if (unlikely(rc != 0)) { 1167 ena_log(adapter->pdev, ERR, 1168 "Failed to configure device after setting new drbr size: %u. Reverting old value: %u and triggering the reset\n", 1169 new_buf_ring_size, old_buf_ring_size); 1170 1171 /* Revert old size and trigger the reset */ 1172 adapter->buf_ring_size = old_buf_ring_size; 1173 ena_free_all_io_rings_resources(adapter); 1174 ena_init_io_rings_advanced(adapter); 1175 1176 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, 1177 adapter); 1178 ena_trigger_reset(adapter, ENA_REGS_RESET_OS_TRIGGER); 1179 1180 } 1181 } 1182 1183 return (rc); 1184 } 1185 1186 int 1187 ena_update_queue_size(struct ena_adapter *adapter, uint32_t new_tx_size, 1188 uint32_t new_rx_size) 1189 { 1190 uint32_t old_tx_size, old_rx_size; 1191 int rc = 0; 1192 bool dev_was_up; 1193 1194 old_tx_size = adapter->requested_tx_ring_size; 1195 old_rx_size = adapter->requested_rx_ring_size; 1196 adapter->requested_tx_ring_size = new_tx_size; 1197 adapter->requested_rx_ring_size = new_rx_size; 1198 1199 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1200 ena_down(adapter); 1201 1202 /* Configure queues with new size. */ 1203 ena_init_io_rings_basic(adapter); 1204 if (dev_was_up) { 1205 rc = ena_up(adapter); 1206 if (unlikely(rc != 0)) { 1207 ena_log(adapter->pdev, ERR, 1208 "Failed to configure device with the new sizes - Tx: %u Rx: %u. Reverting old values - Tx: %u Rx: %u\n", 1209 new_tx_size, new_rx_size, old_tx_size, old_rx_size); 1210 1211 /* Revert old size. */ 1212 adapter->requested_tx_ring_size = old_tx_size; 1213 adapter->requested_rx_ring_size = old_rx_size; 1214 ena_init_io_rings_basic(adapter); 1215 1216 /* And try again. */ 1217 rc = ena_up(adapter); 1218 if (unlikely(rc != 0)) { 1219 ena_log(adapter->pdev, ERR, 1220 "Failed to revert old queue sizes. Triggering device reset.\n"); 1221 /* 1222 * If we've failed again, something had to go 1223 * wrong. After reset, the device should try to 1224 * go up 1225 */ 1226 ENA_FLAG_SET_ATOMIC( 1227 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 1228 ena_trigger_reset(adapter, 1229 ENA_REGS_RESET_OS_TRIGGER); 1230 } 1231 } 1232 } 1233 1234 return (rc); 1235 } 1236 1237 static void 1238 ena_update_io_rings(struct ena_adapter *adapter, uint32_t num) 1239 { 1240 ena_free_all_io_rings_resources(adapter); 1241 /* Force indirection table to be reinitialized */ 1242 ena_com_rss_destroy(adapter->ena_dev); 1243 1244 adapter->num_io_queues = num; 1245 ena_init_io_rings(adapter); 1246 } 1247 1248 /* Caller should sanitize new_num */ 1249 int 1250 ena_update_io_queue_nb(struct ena_adapter *adapter, uint32_t new_num) 1251 { 1252 uint32_t old_num; 1253 int rc = 0; 1254 bool dev_was_up; 1255 1256 dev_was_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 1257 old_num = adapter->num_io_queues; 1258 ena_down(adapter); 1259 1260 ena_update_io_rings(adapter, new_num); 1261 1262 if (dev_was_up) { 1263 rc = ena_up(adapter); 1264 if (unlikely(rc != 0)) { 1265 ena_log(adapter->pdev, ERR, 1266 "Failed to configure device with %u IO queues. " 1267 "Reverting to previous value: %u\n", 1268 new_num, old_num); 1269 1270 ena_update_io_rings(adapter, old_num); 1271 1272 rc = ena_up(adapter); 1273 if (unlikely(rc != 0)) { 1274 ena_log(adapter->pdev, ERR, 1275 "Failed to revert to previous setup IO " 1276 "queues. Triggering device reset.\n"); 1277 ENA_FLAG_SET_ATOMIC( 1278 ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 1279 ena_trigger_reset(adapter, 1280 ENA_REGS_RESET_OS_TRIGGER); 1281 } 1282 } 1283 } 1284 1285 return (rc); 1286 } 1287 1288 static void 1289 ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid) 1290 { 1291 struct ena_ring *rx_ring = &adapter->rx_ring[qid]; 1292 unsigned int i; 1293 1294 for (i = 0; i < rx_ring->ring_size; i++) { 1295 struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i]; 1296 1297 if (rx_info->mbuf != NULL) 1298 ena_free_rx_mbuf(adapter, rx_ring, rx_info); 1299 #ifdef DEV_NETMAP 1300 if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) && 1301 (adapter->ifp->if_capenable & IFCAP_NETMAP)) { 1302 if (rx_info->netmap_buf_idx != 0) 1303 ena_netmap_free_rx_slot(adapter, rx_ring, 1304 rx_info); 1305 } 1306 #endif /* DEV_NETMAP */ 1307 } 1308 } 1309 1310 /** 1311 * ena_refill_all_rx_bufs - allocate all queues Rx buffers 1312 * @adapter: network interface device structure 1313 * 1314 */ 1315 static void 1316 ena_refill_all_rx_bufs(struct ena_adapter *adapter) 1317 { 1318 struct ena_ring *rx_ring; 1319 int i, rc, bufs_num; 1320 1321 for (i = 0; i < adapter->num_io_queues; i++) { 1322 rx_ring = &adapter->rx_ring[i]; 1323 bufs_num = rx_ring->ring_size - 1; 1324 rc = ena_refill_rx_bufs(rx_ring, bufs_num); 1325 if (unlikely(rc != bufs_num)) 1326 ena_log_io(adapter->pdev, WARN, 1327 "refilling Queue %d failed. " 1328 "Allocated %d buffers from: %d\n", i, rc, bufs_num); 1329 #ifdef DEV_NETMAP 1330 rx_ring->initialized = true; 1331 #endif /* DEV_NETMAP */ 1332 } 1333 } 1334 1335 static void 1336 ena_free_all_rx_bufs(struct ena_adapter *adapter) 1337 { 1338 int i; 1339 1340 for (i = 0; i < adapter->num_io_queues; i++) 1341 ena_free_rx_bufs(adapter, i); 1342 } 1343 1344 /** 1345 * ena_free_tx_bufs - Free Tx Buffers per Queue 1346 * @adapter: network interface device structure 1347 * @qid: queue index 1348 **/ 1349 static void 1350 ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid) 1351 { 1352 bool print_once = true; 1353 struct ena_ring *tx_ring = &adapter->tx_ring[qid]; 1354 1355 ENA_RING_MTX_LOCK(tx_ring); 1356 for (int i = 0; i < tx_ring->ring_size; i++) { 1357 struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; 1358 1359 if (tx_info->mbuf == NULL) 1360 continue; 1361 1362 if (print_once) { 1363 ena_log(adapter->pdev, WARN, 1364 "free uncompleted tx mbuf qid %d idx 0x%x\n", 1365 qid, i); 1366 print_once = false; 1367 } else { 1368 ena_log(adapter->pdev, DBG, 1369 "free uncompleted tx mbuf qid %d idx 0x%x\n", 1370 qid, i); 1371 } 1372 1373 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap, 1374 BUS_DMASYNC_POSTWRITE); 1375 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap); 1376 1377 m_free(tx_info->mbuf); 1378 tx_info->mbuf = NULL; 1379 } 1380 ENA_RING_MTX_UNLOCK(tx_ring); 1381 } 1382 1383 static void 1384 ena_free_all_tx_bufs(struct ena_adapter *adapter) 1385 { 1386 1387 for (int i = 0; i < adapter->num_io_queues; i++) 1388 ena_free_tx_bufs(adapter, i); 1389 } 1390 1391 static void 1392 ena_destroy_all_tx_queues(struct ena_adapter *adapter) 1393 { 1394 uint16_t ena_qid; 1395 int i; 1396 1397 for (i = 0; i < adapter->num_io_queues; i++) { 1398 ena_qid = ENA_IO_TXQ_IDX(i); 1399 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1400 } 1401 } 1402 1403 static void 1404 ena_destroy_all_rx_queues(struct ena_adapter *adapter) 1405 { 1406 uint16_t ena_qid; 1407 int i; 1408 1409 for (i = 0; i < adapter->num_io_queues; i++) { 1410 ena_qid = ENA_IO_RXQ_IDX(i); 1411 ena_com_destroy_io_queue(adapter->ena_dev, ena_qid); 1412 } 1413 } 1414 1415 static void 1416 ena_destroy_all_io_queues(struct ena_adapter *adapter) 1417 { 1418 struct ena_que *queue; 1419 int i; 1420 1421 for (i = 0; i < adapter->num_io_queues; i++) { 1422 queue = &adapter->que[i]; 1423 while (taskqueue_cancel(queue->cleanup_tq, 1424 &queue->cleanup_task, NULL)) 1425 taskqueue_drain(queue->cleanup_tq, 1426 &queue->cleanup_task); 1427 taskqueue_free(queue->cleanup_tq); 1428 } 1429 1430 ena_destroy_all_tx_queues(adapter); 1431 ena_destroy_all_rx_queues(adapter); 1432 } 1433 1434 static int 1435 ena_create_io_queues(struct ena_adapter *adapter) 1436 { 1437 struct ena_com_dev *ena_dev = adapter->ena_dev; 1438 struct ena_com_create_io_ctx ctx; 1439 struct ena_ring *ring; 1440 struct ena_que *queue; 1441 uint16_t ena_qid; 1442 uint32_t msix_vector; 1443 cpuset_t *cpu_mask = NULL; 1444 int rc, i; 1445 1446 /* Create TX queues */ 1447 for (i = 0; i < adapter->num_io_queues; i++) { 1448 msix_vector = ENA_IO_IRQ_IDX(i); 1449 ena_qid = ENA_IO_TXQ_IDX(i); 1450 ctx.mem_queue_type = ena_dev->tx_mem_queue_type; 1451 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX; 1452 ctx.queue_size = adapter->requested_tx_ring_size; 1453 ctx.msix_vector = msix_vector; 1454 ctx.qid = ena_qid; 1455 ctx.numa_node = adapter->que[i].domain; 1456 1457 rc = ena_com_create_io_queue(ena_dev, &ctx); 1458 if (rc != 0) { 1459 ena_log(adapter->pdev, ERR, 1460 "Failed to create io TX queue #%d rc: %d\n", i, rc); 1461 goto err_tx; 1462 } 1463 ring = &adapter->tx_ring[i]; 1464 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1465 &ring->ena_com_io_sq, 1466 &ring->ena_com_io_cq); 1467 if (rc != 0) { 1468 ena_log(adapter->pdev, ERR, 1469 "Failed to get TX queue handlers. TX queue num" 1470 " %d rc: %d\n", i, rc); 1471 ena_com_destroy_io_queue(ena_dev, ena_qid); 1472 goto err_tx; 1473 } 1474 1475 if (ctx.numa_node >= 0) { 1476 ena_com_update_numa_node(ring->ena_com_io_cq, 1477 ctx.numa_node); 1478 } 1479 } 1480 1481 /* Create RX queues */ 1482 for (i = 0; i < adapter->num_io_queues; i++) { 1483 msix_vector = ENA_IO_IRQ_IDX(i); 1484 ena_qid = ENA_IO_RXQ_IDX(i); 1485 ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 1486 ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX; 1487 ctx.queue_size = adapter->requested_rx_ring_size; 1488 ctx.msix_vector = msix_vector; 1489 ctx.qid = ena_qid; 1490 ctx.numa_node = adapter->que[i].domain; 1491 1492 rc = ena_com_create_io_queue(ena_dev, &ctx); 1493 if (unlikely(rc != 0)) { 1494 ena_log(adapter->pdev, ERR, 1495 "Failed to create io RX queue[%d] rc: %d\n", i, rc); 1496 goto err_rx; 1497 } 1498 1499 ring = &adapter->rx_ring[i]; 1500 rc = ena_com_get_io_handlers(ena_dev, ena_qid, 1501 &ring->ena_com_io_sq, 1502 &ring->ena_com_io_cq); 1503 if (unlikely(rc != 0)) { 1504 ena_log(adapter->pdev, ERR, 1505 "Failed to get RX queue handlers. RX queue num" 1506 " %d rc: %d\n", i, rc); 1507 ena_com_destroy_io_queue(ena_dev, ena_qid); 1508 goto err_rx; 1509 } 1510 1511 if (ctx.numa_node >= 0) { 1512 ena_com_update_numa_node(ring->ena_com_io_cq, 1513 ctx.numa_node); 1514 } 1515 } 1516 1517 for (i = 0; i < adapter->num_io_queues; i++) { 1518 queue = &adapter->que[i]; 1519 1520 NET_TASK_INIT(&queue->cleanup_task, 0, ena_cleanup, queue); 1521 queue->cleanup_tq = taskqueue_create_fast("ena cleanup", 1522 M_WAITOK, taskqueue_thread_enqueue, &queue->cleanup_tq); 1523 1524 #ifdef RSS 1525 cpu_mask = &queue->cpu_mask; 1526 #endif 1527 taskqueue_start_threads_cpuset(&queue->cleanup_tq, 1, PI_NET, 1528 cpu_mask, 1529 "%s queue %d cleanup", 1530 device_get_nameunit(adapter->pdev), i); 1531 } 1532 1533 return (0); 1534 1535 err_rx: 1536 while (i--) 1537 ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i)); 1538 i = adapter->num_io_queues; 1539 err_tx: 1540 while (i--) 1541 ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i)); 1542 1543 return (ENXIO); 1544 } 1545 1546 /********************************************************************* 1547 * 1548 * MSIX & Interrupt Service routine 1549 * 1550 **********************************************************************/ 1551 1552 /** 1553 * ena_handle_msix - MSIX Interrupt Handler for admin/async queue 1554 * @arg: interrupt number 1555 **/ 1556 static void 1557 ena_intr_msix_mgmnt(void *arg) 1558 { 1559 struct ena_adapter *adapter = (struct ena_adapter *)arg; 1560 1561 ena_com_admin_q_comp_intr_handler(adapter->ena_dev); 1562 if (likely(ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter))) 1563 ena_com_aenq_intr_handler(adapter->ena_dev, arg); 1564 } 1565 1566 /** 1567 * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx 1568 * @arg: queue 1569 **/ 1570 static int 1571 ena_handle_msix(void *arg) 1572 { 1573 struct ena_que *queue = arg; 1574 struct ena_adapter *adapter = queue->adapter; 1575 if_t ifp = adapter->ifp; 1576 1577 if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)) 1578 return (FILTER_STRAY); 1579 1580 taskqueue_enqueue(queue->cleanup_tq, &queue->cleanup_task); 1581 1582 return (FILTER_HANDLED); 1583 } 1584 1585 static int 1586 ena_enable_msix(struct ena_adapter *adapter) 1587 { 1588 device_t dev = adapter->pdev; 1589 int msix_vecs, msix_req; 1590 int i, rc = 0; 1591 1592 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1593 ena_log(dev, ERR, "Error, MSI-X is already enabled\n"); 1594 return (EINVAL); 1595 } 1596 1597 /* Reserved the max msix vectors we might need */ 1598 msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues); 1599 1600 adapter->msix_entries = malloc(msix_vecs * sizeof(struct msix_entry), 1601 M_DEVBUF, M_WAITOK | M_ZERO); 1602 1603 ena_log(dev, DBG, "trying to enable MSI-X, vectors: %d\n", 1604 msix_vecs); 1605 1606 for (i = 0; i < msix_vecs; i++) { 1607 adapter->msix_entries[i].entry = i; 1608 /* Vectors must start from 1 */ 1609 adapter->msix_entries[i].vector = i + 1; 1610 } 1611 1612 msix_req = msix_vecs; 1613 rc = pci_alloc_msix(dev, &msix_vecs); 1614 if (unlikely(rc != 0)) { 1615 ena_log(dev, ERR, 1616 "Failed to enable MSIX, vectors %d rc %d\n", msix_vecs, rc); 1617 1618 rc = ENOSPC; 1619 goto err_msix_free; 1620 } 1621 1622 if (msix_vecs != msix_req) { 1623 if (msix_vecs == ENA_ADMIN_MSIX_VEC) { 1624 ena_log(dev, ERR, 1625 "Not enough number of MSI-x allocated: %d\n", 1626 msix_vecs); 1627 pci_release_msi(dev); 1628 rc = ENOSPC; 1629 goto err_msix_free; 1630 } 1631 ena_log(dev, ERR, "Enable only %d MSI-x (out of %d), reduce " 1632 "the number of queues\n", msix_vecs, msix_req); 1633 } 1634 1635 adapter->msix_vecs = msix_vecs; 1636 ENA_FLAG_SET_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1637 1638 return (0); 1639 1640 err_msix_free: 1641 free(adapter->msix_entries, M_DEVBUF); 1642 adapter->msix_entries = NULL; 1643 1644 return (rc); 1645 } 1646 1647 static void 1648 ena_setup_mgmnt_intr(struct ena_adapter *adapter) 1649 { 1650 1651 snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name, 1652 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s", 1653 device_get_nameunit(adapter->pdev)); 1654 /* 1655 * Handler is NULL on purpose, it will be set 1656 * when mgmnt interrupt is acquired 1657 */ 1658 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL; 1659 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter; 1660 adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector = 1661 adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector; 1662 } 1663 1664 static int 1665 ena_setup_io_intr(struct ena_adapter *adapter) 1666 { 1667 #ifdef RSS 1668 int num_buckets = rss_getnumbuckets(); 1669 static int last_bind = 0; 1670 int cur_bind; 1671 int idx; 1672 #endif 1673 int irq_idx; 1674 1675 if (adapter->msix_entries == NULL) 1676 return (EINVAL); 1677 1678 #ifdef RSS 1679 if (adapter->first_bind < 0) { 1680 adapter->first_bind = last_bind; 1681 last_bind = (last_bind + adapter->num_io_queues) % num_buckets; 1682 } 1683 cur_bind = adapter->first_bind; 1684 #endif 1685 1686 for (int i = 0; i < adapter->num_io_queues; i++) { 1687 irq_idx = ENA_IO_IRQ_IDX(i); 1688 1689 snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE, 1690 "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i); 1691 adapter->irq_tbl[irq_idx].handler = ena_handle_msix; 1692 adapter->irq_tbl[irq_idx].data = &adapter->que[i]; 1693 adapter->irq_tbl[irq_idx].vector = 1694 adapter->msix_entries[irq_idx].vector; 1695 ena_log(adapter->pdev, DBG, "ena_setup_io_intr vector: %d\n", 1696 adapter->msix_entries[irq_idx].vector); 1697 1698 #ifdef RSS 1699 adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu = 1700 rss_getcpu(cur_bind); 1701 cur_bind = (cur_bind + 1) % num_buckets; 1702 CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask); 1703 1704 for (idx = 0; idx < MAXMEMDOM; ++idx) { 1705 if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx])) 1706 break; 1707 } 1708 adapter->que[i].domain = idx; 1709 #else 1710 adapter->que[i].domain = -1; 1711 #endif 1712 } 1713 1714 return (0); 1715 } 1716 1717 static int 1718 ena_request_mgmnt_irq(struct ena_adapter *adapter) 1719 { 1720 device_t pdev = adapter->pdev; 1721 struct ena_irq *irq; 1722 unsigned long flags; 1723 int rc, rcc; 1724 1725 flags = RF_ACTIVE | RF_SHAREABLE; 1726 1727 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1728 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1729 &irq->vector, flags); 1730 1731 if (unlikely(irq->res == NULL)) { 1732 ena_log(pdev, ERR, "could not allocate irq vector: %d\n", 1733 irq->vector); 1734 return (ENXIO); 1735 } 1736 1737 rc = bus_setup_intr(adapter->pdev, irq->res, 1738 INTR_TYPE_NET | INTR_MPSAFE, NULL, ena_intr_msix_mgmnt, 1739 irq->data, &irq->cookie); 1740 if (unlikely(rc != 0)) { 1741 ena_log(pdev, ERR, "failed to register " 1742 "interrupt handler for irq %ju: %d\n", 1743 rman_get_start(irq->res), rc); 1744 goto err_res_free; 1745 } 1746 irq->requested = true; 1747 1748 return (rc); 1749 1750 err_res_free: 1751 ena_log(pdev, INFO, "releasing resource for irq %d\n", irq->vector); 1752 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1753 irq->vector, irq->res); 1754 if (unlikely(rcc != 0)) 1755 ena_log(pdev, ERR, "dev has no parent while " 1756 "releasing res for irq: %d\n", irq->vector); 1757 irq->res = NULL; 1758 1759 return (rc); 1760 } 1761 1762 static int 1763 ena_request_io_irq(struct ena_adapter *adapter) 1764 { 1765 device_t pdev = adapter->pdev; 1766 struct ena_irq *irq; 1767 unsigned long flags = 0; 1768 int rc = 0, i, rcc; 1769 1770 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter))) { 1771 ena_log(pdev, ERR, 1772 "failed to request I/O IRQ: MSI-X is not enabled\n"); 1773 return (EINVAL); 1774 } else { 1775 flags = RF_ACTIVE | RF_SHAREABLE; 1776 } 1777 1778 for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1779 irq = &adapter->irq_tbl[i]; 1780 1781 if (unlikely(irq->requested)) 1782 continue; 1783 1784 irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ, 1785 &irq->vector, flags); 1786 if (unlikely(irq->res == NULL)) { 1787 rc = ENOMEM; 1788 ena_log(pdev, ERR, "could not allocate irq vector: %d\n", 1789 irq->vector); 1790 goto err; 1791 } 1792 1793 rc = bus_setup_intr(adapter->pdev, irq->res, 1794 INTR_TYPE_NET | INTR_MPSAFE, irq->handler, NULL, 1795 irq->data, &irq->cookie); 1796 if (unlikely(rc != 0)) { 1797 ena_log(pdev, ERR, "failed to register " 1798 "interrupt handler for irq %ju: %d\n", 1799 rman_get_start(irq->res), rc); 1800 goto err; 1801 } 1802 irq->requested = true; 1803 1804 #ifdef RSS 1805 rc = bus_bind_intr(adapter->pdev, irq->res, irq->cpu); 1806 if (unlikely(rc != 0)) { 1807 ena_log(pdev, ERR, "failed to bind " 1808 "interrupt handler for irq %ju to cpu %d: %d\n", 1809 rman_get_start(irq->res), irq->cpu, rc); 1810 goto err; 1811 } 1812 1813 ena_log(pdev, INFO, "queue %d - cpu %d\n", 1814 i - ENA_IO_IRQ_FIRST_IDX, irq->cpu); 1815 #endif 1816 } 1817 1818 return (rc); 1819 1820 err: 1821 1822 for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) { 1823 irq = &adapter->irq_tbl[i]; 1824 rcc = 0; 1825 1826 /* Once we entered err: section and irq->requested is true we 1827 free both intr and resources */ 1828 if (irq->requested) 1829 rcc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); 1830 if (unlikely(rcc != 0)) 1831 ena_log(pdev, ERR, "could not release irq: %d, error: %d\n", 1832 irq->vector, rcc); 1833 1834 /* If we entred err: section without irq->requested set we know 1835 it was bus_alloc_resource_any() that needs cleanup, provided 1836 res is not NULL. In case res is NULL no work in needed in 1837 this iteration */ 1838 rcc = 0; 1839 if (irq->res != NULL) { 1840 rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1841 irq->vector, irq->res); 1842 } 1843 if (unlikely(rcc != 0)) 1844 ena_log(pdev, ERR, "dev has no parent while " 1845 "releasing res for irq: %d\n", irq->vector); 1846 irq->requested = false; 1847 irq->res = NULL; 1848 } 1849 1850 return (rc); 1851 } 1852 1853 static void 1854 ena_free_mgmnt_irq(struct ena_adapter *adapter) 1855 { 1856 device_t pdev = adapter->pdev; 1857 struct ena_irq *irq; 1858 int rc; 1859 1860 irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX]; 1861 if (irq->requested) { 1862 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector); 1863 rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie); 1864 if (unlikely(rc != 0)) 1865 ena_log(pdev, ERR, "failed to tear down irq: %d\n", 1866 irq->vector); 1867 irq->requested = 0; 1868 } 1869 1870 if (irq->res != NULL) { 1871 ena_log(pdev, DBG, "release resource irq: %d\n", irq->vector); 1872 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1873 irq->vector, irq->res); 1874 irq->res = NULL; 1875 if (unlikely(rc != 0)) 1876 ena_log(pdev, ERR, "dev has no parent while " 1877 "releasing res for irq: %d\n", irq->vector); 1878 } 1879 } 1880 1881 static void 1882 ena_free_io_irq(struct ena_adapter *adapter) 1883 { 1884 device_t pdev = adapter->pdev; 1885 struct ena_irq *irq; 1886 int rc; 1887 1888 for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) { 1889 irq = &adapter->irq_tbl[i]; 1890 if (irq->requested) { 1891 ena_log(pdev, DBG, "tear down irq: %d\n", irq->vector); 1892 rc = bus_teardown_intr(adapter->pdev, irq->res, 1893 irq->cookie); 1894 if (unlikely(rc != 0)) { 1895 ena_log(pdev, ERR, "failed to tear down irq: %d\n", 1896 irq->vector); 1897 } 1898 irq->requested = 0; 1899 } 1900 1901 if (irq->res != NULL) { 1902 ena_log(pdev, DBG, "release resource irq: %d\n", 1903 irq->vector); 1904 rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ, 1905 irq->vector, irq->res); 1906 irq->res = NULL; 1907 if (unlikely(rc != 0)) { 1908 ena_log(pdev, ERR, "dev has no parent" 1909 " while releasing res for irq: %d\n", 1910 irq->vector); 1911 } 1912 } 1913 } 1914 } 1915 1916 static void 1917 ena_free_irqs(struct ena_adapter* adapter) 1918 { 1919 1920 ena_free_io_irq(adapter); 1921 ena_free_mgmnt_irq(adapter); 1922 ena_disable_msix(adapter); 1923 } 1924 1925 static void 1926 ena_disable_msix(struct ena_adapter *adapter) 1927 { 1928 1929 if (ENA_FLAG_ISSET(ENA_FLAG_MSIX_ENABLED, adapter)) { 1930 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_MSIX_ENABLED, adapter); 1931 pci_release_msi(adapter->pdev); 1932 } 1933 1934 adapter->msix_vecs = 0; 1935 free(adapter->msix_entries, M_DEVBUF); 1936 adapter->msix_entries = NULL; 1937 } 1938 1939 static void 1940 ena_unmask_all_io_irqs(struct ena_adapter *adapter) 1941 { 1942 struct ena_com_io_cq* io_cq; 1943 struct ena_eth_io_intr_reg intr_reg; 1944 struct ena_ring *tx_ring; 1945 uint16_t ena_qid; 1946 int i; 1947 1948 /* Unmask interrupts for all queues */ 1949 for (i = 0; i < adapter->num_io_queues; i++) { 1950 ena_qid = ENA_IO_TXQ_IDX(i); 1951 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid]; 1952 ena_com_update_intr_reg(&intr_reg, 0, 0, true); 1953 tx_ring = &adapter->tx_ring[i]; 1954 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1); 1955 ena_com_unmask_intr(io_cq, &intr_reg); 1956 } 1957 } 1958 1959 static int 1960 ena_up_complete(struct ena_adapter *adapter) 1961 { 1962 int rc; 1963 1964 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) { 1965 rc = ena_rss_configure(adapter); 1966 if (rc != 0) { 1967 ena_log(adapter->pdev, ERR, 1968 "Failed to configure RSS\n"); 1969 return (rc); 1970 } 1971 } 1972 1973 rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu); 1974 if (unlikely(rc != 0)) 1975 return (rc); 1976 1977 ena_refill_all_rx_bufs(adapter); 1978 ena_reset_counters((counter_u64_t *)&adapter->hw_stats, 1979 sizeof(adapter->hw_stats)); 1980 1981 return (0); 1982 } 1983 1984 static void 1985 set_io_rings_size(struct ena_adapter *adapter, int new_tx_size, 1986 int new_rx_size) 1987 { 1988 int i; 1989 1990 for (i = 0; i < adapter->num_io_queues; i++) { 1991 adapter->tx_ring[i].ring_size = new_tx_size; 1992 adapter->rx_ring[i].ring_size = new_rx_size; 1993 } 1994 } 1995 1996 static int 1997 create_queues_with_size_backoff(struct ena_adapter *adapter) 1998 { 1999 device_t pdev = adapter->pdev; 2000 int rc; 2001 uint32_t cur_rx_ring_size, cur_tx_ring_size; 2002 uint32_t new_rx_ring_size, new_tx_ring_size; 2003 2004 /* 2005 * Current queue sizes might be set to smaller than the requested 2006 * ones due to past queue allocation failures. 2007 */ 2008 set_io_rings_size(adapter, adapter->requested_tx_ring_size, 2009 adapter->requested_rx_ring_size); 2010 2011 while (1) { 2012 /* Allocate transmit descriptors */ 2013 rc = ena_setup_all_tx_resources(adapter); 2014 if (unlikely(rc != 0)) { 2015 ena_log(pdev, ERR, "err_setup_tx\n"); 2016 goto err_setup_tx; 2017 } 2018 2019 /* Allocate receive descriptors */ 2020 rc = ena_setup_all_rx_resources(adapter); 2021 if (unlikely(rc != 0)) { 2022 ena_log(pdev, ERR, "err_setup_rx\n"); 2023 goto err_setup_rx; 2024 } 2025 2026 /* Create IO queues for Rx & Tx */ 2027 rc = ena_create_io_queues(adapter); 2028 if (unlikely(rc != 0)) { 2029 ena_log(pdev, ERR, 2030 "create IO queues failed\n"); 2031 goto err_io_que; 2032 } 2033 2034 return (0); 2035 2036 err_io_que: 2037 ena_free_all_rx_resources(adapter); 2038 err_setup_rx: 2039 ena_free_all_tx_resources(adapter); 2040 err_setup_tx: 2041 /* 2042 * Lower the ring size if ENOMEM. Otherwise, return the 2043 * error straightaway. 2044 */ 2045 if (unlikely(rc != ENOMEM)) { 2046 ena_log(pdev, ERR, 2047 "Queue creation failed with error code: %d\n", rc); 2048 return (rc); 2049 } 2050 2051 cur_tx_ring_size = adapter->tx_ring[0].ring_size; 2052 cur_rx_ring_size = adapter->rx_ring[0].ring_size; 2053 2054 ena_log(pdev, ERR, 2055 "Not enough memory to create queues with sizes TX=%d, RX=%d\n", 2056 cur_tx_ring_size, cur_rx_ring_size); 2057 2058 new_tx_ring_size = cur_tx_ring_size; 2059 new_rx_ring_size = cur_rx_ring_size; 2060 2061 /* 2062 * Decrease the size of a larger queue, or decrease both if they are 2063 * the same size. 2064 */ 2065 if (cur_rx_ring_size <= cur_tx_ring_size) 2066 new_tx_ring_size = cur_tx_ring_size / 2; 2067 if (cur_rx_ring_size >= cur_tx_ring_size) 2068 new_rx_ring_size = cur_rx_ring_size / 2; 2069 2070 if (new_tx_ring_size < ENA_MIN_RING_SIZE || 2071 new_rx_ring_size < ENA_MIN_RING_SIZE) { 2072 ena_log(pdev, ERR, 2073 "Queue creation failed with the smallest possible queue size" 2074 "of %d for both queues. Not retrying with smaller queues\n", 2075 ENA_MIN_RING_SIZE); 2076 return (rc); 2077 } 2078 2079 ena_log(pdev, INFO, 2080 "Retrying queue creation with sizes TX=%d, RX=%d\n", 2081 new_tx_ring_size, new_rx_ring_size); 2082 2083 set_io_rings_size(adapter, new_tx_ring_size, new_rx_ring_size); 2084 } 2085 } 2086 2087 int 2088 ena_up(struct ena_adapter *adapter) 2089 { 2090 int rc = 0; 2091 2092 ENA_LOCK_ASSERT(); 2093 2094 if (unlikely(device_is_attached(adapter->pdev) == 0)) { 2095 ena_log(adapter->pdev, ERR, "device is not attached!\n"); 2096 return (ENXIO); 2097 } 2098 2099 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 2100 return (0); 2101 2102 ena_log(adapter->pdev, INFO, "device is going UP\n"); 2103 2104 /* 2105 * ena_timer_service can use functions, which write to the admin queue. 2106 * Those calls are not protected by ENA_LOCK, and because of that, the 2107 * timer should be stopped when bringing the device up or down. 2108 */ 2109 ENA_TIMER_DRAIN(adapter); 2110 2111 /* setup interrupts for IO queues */ 2112 rc = ena_setup_io_intr(adapter); 2113 if (unlikely(rc != 0)) { 2114 ena_log(adapter->pdev, ERR, "error setting up IO interrupt\n"); 2115 goto error; 2116 } 2117 rc = ena_request_io_irq(adapter); 2118 if (unlikely(rc != 0)) { 2119 ena_log(adapter->pdev, ERR, "err_req_irq\n"); 2120 goto error; 2121 } 2122 2123 ena_log(adapter->pdev, INFO, 2124 "Creating %u IO queues. Rx queue size: %d, Tx queue size: %d, " 2125 "LLQ is %s\n", 2126 adapter->num_io_queues, 2127 adapter->requested_rx_ring_size, 2128 adapter->requested_tx_ring_size, 2129 (adapter->ena_dev->tx_mem_queue_type == 2130 ENA_ADMIN_PLACEMENT_POLICY_DEV) ? "ENABLED" : "DISABLED"); 2131 2132 rc = create_queues_with_size_backoff(adapter); 2133 if (unlikely(rc != 0)) { 2134 ena_log(adapter->pdev, ERR, 2135 "error creating queues with size backoff\n"); 2136 goto err_create_queues_with_backoff; 2137 } 2138 2139 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 2140 if_link_state_change(adapter->ifp, LINK_STATE_UP); 2141 2142 rc = ena_up_complete(adapter); 2143 if (unlikely(rc != 0)) 2144 goto err_up_complete; 2145 2146 counter_u64_add(adapter->dev_stats.interface_up, 1); 2147 2148 ena_update_hwassist(adapter); 2149 2150 if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING, 2151 IFF_DRV_OACTIVE); 2152 2153 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2154 2155 ena_unmask_all_io_irqs(adapter); 2156 2157 ENA_TIMER_RESET(adapter); 2158 2159 return (0); 2160 2161 err_up_complete: 2162 ena_destroy_all_io_queues(adapter); 2163 ena_free_all_rx_resources(adapter); 2164 ena_free_all_tx_resources(adapter); 2165 err_create_queues_with_backoff: 2166 ena_free_io_irq(adapter); 2167 error: 2168 ENA_TIMER_RESET(adapter); 2169 2170 return (rc); 2171 } 2172 2173 static uint64_t 2174 ena_get_counter(if_t ifp, ift_counter cnt) 2175 { 2176 struct ena_adapter *adapter; 2177 struct ena_hw_stats *stats; 2178 2179 adapter = if_getsoftc(ifp); 2180 stats = &adapter->hw_stats; 2181 2182 switch (cnt) { 2183 case IFCOUNTER_IPACKETS: 2184 return (counter_u64_fetch(stats->rx_packets)); 2185 case IFCOUNTER_OPACKETS: 2186 return (counter_u64_fetch(stats->tx_packets)); 2187 case IFCOUNTER_IBYTES: 2188 return (counter_u64_fetch(stats->rx_bytes)); 2189 case IFCOUNTER_OBYTES: 2190 return (counter_u64_fetch(stats->tx_bytes)); 2191 case IFCOUNTER_IQDROPS: 2192 return (counter_u64_fetch(stats->rx_drops)); 2193 case IFCOUNTER_OQDROPS: 2194 return (counter_u64_fetch(stats->tx_drops)); 2195 default: 2196 return (if_get_counter_default(ifp, cnt)); 2197 } 2198 } 2199 2200 static int 2201 ena_media_change(if_t ifp) 2202 { 2203 /* Media Change is not supported by firmware */ 2204 return (0); 2205 } 2206 2207 static void 2208 ena_media_status(if_t ifp, struct ifmediareq *ifmr) 2209 { 2210 struct ena_adapter *adapter = if_getsoftc(ifp); 2211 ena_log(adapter->pdev, DBG, "Media status update\n"); 2212 2213 ENA_LOCK_LOCK(); 2214 2215 ifmr->ifm_status = IFM_AVALID; 2216 ifmr->ifm_active = IFM_ETHER; 2217 2218 if (!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) { 2219 ENA_LOCK_UNLOCK(); 2220 ena_log(adapter->pdev, INFO, "Link is down\n"); 2221 return; 2222 } 2223 2224 ifmr->ifm_status |= IFM_ACTIVE; 2225 ifmr->ifm_active |= IFM_UNKNOWN | IFM_FDX; 2226 2227 ENA_LOCK_UNLOCK(); 2228 } 2229 2230 static void 2231 ena_init(void *arg) 2232 { 2233 struct ena_adapter *adapter = (struct ena_adapter *)arg; 2234 2235 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) { 2236 ENA_LOCK_LOCK(); 2237 ena_up(adapter); 2238 ENA_LOCK_UNLOCK(); 2239 } 2240 } 2241 2242 static int 2243 ena_ioctl(if_t ifp, u_long command, caddr_t data) 2244 { 2245 struct ena_adapter *adapter; 2246 struct ifreq *ifr; 2247 int rc; 2248 2249 adapter = ifp->if_softc; 2250 ifr = (struct ifreq *)data; 2251 2252 /* 2253 * Acquiring lock to prevent from running up and down routines parallel. 2254 */ 2255 rc = 0; 2256 switch (command) { 2257 case SIOCSIFMTU: 2258 if (ifp->if_mtu == ifr->ifr_mtu) 2259 break; 2260 ENA_LOCK_LOCK(); 2261 ena_down(adapter); 2262 2263 ena_change_mtu(ifp, ifr->ifr_mtu); 2264 2265 rc = ena_up(adapter); 2266 ENA_LOCK_UNLOCK(); 2267 break; 2268 2269 case SIOCSIFFLAGS: 2270 if ((ifp->if_flags & IFF_UP) != 0) { 2271 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2272 if ((ifp->if_flags & (IFF_PROMISC | 2273 IFF_ALLMULTI)) != 0) { 2274 ena_log(adapter->pdev, INFO, 2275 "ioctl promisc/allmulti\n"); 2276 } 2277 } else { 2278 ENA_LOCK_LOCK(); 2279 rc = ena_up(adapter); 2280 ENA_LOCK_UNLOCK(); 2281 } 2282 } else { 2283 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) { 2284 ENA_LOCK_LOCK(); 2285 ena_down(adapter); 2286 ENA_LOCK_UNLOCK(); 2287 } 2288 } 2289 break; 2290 2291 case SIOCADDMULTI: 2292 case SIOCDELMULTI: 2293 break; 2294 2295 case SIOCSIFMEDIA: 2296 case SIOCGIFMEDIA: 2297 rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command); 2298 break; 2299 2300 case SIOCSIFCAP: 2301 { 2302 int reinit = 0; 2303 2304 if (ifr->ifr_reqcap != ifp->if_capenable) { 2305 ifp->if_capenable = ifr->ifr_reqcap; 2306 reinit = 1; 2307 } 2308 2309 if ((reinit != 0) && 2310 ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0)) { 2311 ENA_LOCK_LOCK(); 2312 ena_down(adapter); 2313 rc = ena_up(adapter); 2314 ENA_LOCK_UNLOCK(); 2315 } 2316 } 2317 2318 break; 2319 default: 2320 rc = ether_ioctl(ifp, command, data); 2321 break; 2322 } 2323 2324 return (rc); 2325 } 2326 2327 static int 2328 ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat) 2329 { 2330 int caps = 0; 2331 2332 if ((feat->offload.tx & 2333 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2334 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK | 2335 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0) 2336 caps |= IFCAP_TXCSUM; 2337 2338 if ((feat->offload.tx & 2339 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK | 2340 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0) 2341 caps |= IFCAP_TXCSUM_IPV6; 2342 2343 if ((feat->offload.tx & 2344 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0) 2345 caps |= IFCAP_TSO4; 2346 2347 if ((feat->offload.tx & 2348 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0) 2349 caps |= IFCAP_TSO6; 2350 2351 if ((feat->offload.rx_supported & 2352 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK | 2353 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0) 2354 caps |= IFCAP_RXCSUM; 2355 2356 if ((feat->offload.rx_supported & 2357 ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0) 2358 caps |= IFCAP_RXCSUM_IPV6; 2359 2360 caps |= IFCAP_LRO | IFCAP_JUMBO_MTU; 2361 2362 return (caps); 2363 } 2364 2365 static void 2366 ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp) 2367 { 2368 2369 host_info->supported_network_features[0] = 2370 (uint32_t)if_getcapabilities(ifp); 2371 } 2372 2373 static void 2374 ena_update_hwassist(struct ena_adapter *adapter) 2375 { 2376 if_t ifp = adapter->ifp; 2377 uint32_t feat = adapter->tx_offload_cap; 2378 int cap = if_getcapenable(ifp); 2379 int flags = 0; 2380 2381 if_clearhwassist(ifp); 2382 2383 if ((cap & IFCAP_TXCSUM) != 0) { 2384 if ((feat & 2385 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0) 2386 flags |= CSUM_IP; 2387 if ((feat & 2388 (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK | 2389 ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0) 2390 flags |= CSUM_IP_UDP | CSUM_IP_TCP; 2391 } 2392 2393 if ((cap & IFCAP_TXCSUM_IPV6) != 0) 2394 flags |= CSUM_IP6_UDP | CSUM_IP6_TCP; 2395 2396 if ((cap & IFCAP_TSO4) != 0) 2397 flags |= CSUM_IP_TSO; 2398 2399 if ((cap & IFCAP_TSO6) != 0) 2400 flags |= CSUM_IP6_TSO; 2401 2402 if_sethwassistbits(ifp, flags, 0); 2403 } 2404 2405 static int 2406 ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter, 2407 struct ena_com_dev_get_features_ctx *feat) 2408 { 2409 if_t ifp; 2410 int caps = 0; 2411 2412 ifp = adapter->ifp = if_gethandle(IFT_ETHER); 2413 if (unlikely(ifp == NULL)) { 2414 ena_log(pdev, ERR, "can not allocate ifnet structure\n"); 2415 return (ENXIO); 2416 } 2417 if_initname(ifp, device_get_name(pdev), device_get_unit(pdev)); 2418 if_setdev(ifp, pdev); 2419 if_setsoftc(ifp, adapter); 2420 2421 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | 2422 IFF_KNOWSEPOCH); 2423 if_setinitfn(ifp, ena_init); 2424 if_settransmitfn(ifp, ena_mq_start); 2425 if_setqflushfn(ifp, ena_qflush); 2426 if_setioctlfn(ifp, ena_ioctl); 2427 if_setgetcounterfn(ifp, ena_get_counter); 2428 2429 if_setsendqlen(ifp, adapter->requested_tx_ring_size); 2430 if_setsendqready(ifp); 2431 if_setmtu(ifp, ETHERMTU); 2432 if_setbaudrate(ifp, 0); 2433 /* Zeroize capabilities... */ 2434 if_setcapabilities(ifp, 0); 2435 if_setcapenable(ifp, 0); 2436 /* check hardware support */ 2437 caps = ena_get_dev_offloads(feat); 2438 /* ... and set them */ 2439 if_setcapabilitiesbit(ifp, caps, 0); 2440 2441 /* TSO parameters */ 2442 ifp->if_hw_tsomax = ENA_TSO_MAXSIZE - 2443 (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN); 2444 ifp->if_hw_tsomaxsegcount = adapter->max_tx_sgl_size - 1; 2445 ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE; 2446 2447 if_setifheaderlen(ifp, sizeof(struct ether_vlan_header)); 2448 if_setcapenable(ifp, if_getcapabilities(ifp)); 2449 2450 /* 2451 * Specify the media types supported by this adapter and register 2452 * callbacks to update media and link information 2453 */ 2454 ifmedia_init(&adapter->media, IFM_IMASK, 2455 ena_media_change, ena_media_status); 2456 ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL); 2457 ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO); 2458 2459 ether_ifattach(ifp, adapter->mac_addr); 2460 2461 return (0); 2462 } 2463 2464 void 2465 ena_down(struct ena_adapter *adapter) 2466 { 2467 int rc; 2468 2469 ENA_LOCK_ASSERT(); 2470 2471 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 2472 return; 2473 2474 /* Drain timer service to avoid admin queue race condition. */ 2475 ENA_TIMER_DRAIN(adapter); 2476 2477 ena_log(adapter->pdev, INFO, "device is going DOWN\n"); 2478 2479 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP, adapter); 2480 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, 2481 IFF_DRV_RUNNING); 2482 2483 ena_free_io_irq(adapter); 2484 2485 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) { 2486 rc = ena_com_dev_reset(adapter->ena_dev, 2487 adapter->reset_reason); 2488 if (unlikely(rc != 0)) 2489 ena_log(adapter->pdev, ERR, 2490 "Device reset failed\n"); 2491 } 2492 2493 ena_destroy_all_io_queues(adapter); 2494 2495 ena_free_all_tx_bufs(adapter); 2496 ena_free_all_rx_bufs(adapter); 2497 ena_free_all_tx_resources(adapter); 2498 ena_free_all_rx_resources(adapter); 2499 2500 counter_u64_add(adapter->dev_stats.interface_down, 1); 2501 2502 ENA_TIMER_RESET(adapter); 2503 } 2504 2505 static uint32_t 2506 ena_calc_max_io_queue_num(device_t pdev, struct ena_com_dev *ena_dev, 2507 struct ena_com_dev_get_features_ctx *get_feat_ctx) 2508 { 2509 uint32_t io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues; 2510 2511 /* Regular queues capabilities */ 2512 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2513 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2514 &get_feat_ctx->max_queue_ext.max_queue_ext; 2515 io_rx_num = min_t(int, max_queue_ext->max_rx_sq_num, 2516 max_queue_ext->max_rx_cq_num); 2517 2518 io_tx_sq_num = max_queue_ext->max_tx_sq_num; 2519 io_tx_cq_num = max_queue_ext->max_tx_cq_num; 2520 } else { 2521 struct ena_admin_queue_feature_desc *max_queues = 2522 &get_feat_ctx->max_queues; 2523 io_tx_sq_num = max_queues->max_sq_num; 2524 io_tx_cq_num = max_queues->max_cq_num; 2525 io_rx_num = min_t(int, io_tx_sq_num, io_tx_cq_num); 2526 } 2527 2528 /* In case of LLQ use the llq fields for the tx SQ/CQ */ 2529 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 2530 io_tx_sq_num = get_feat_ctx->llq.max_llq_num; 2531 2532 max_num_io_queues = min_t(uint32_t, mp_ncpus, ENA_MAX_NUM_IO_QUEUES); 2533 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_rx_num); 2534 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_sq_num); 2535 max_num_io_queues = min_t(uint32_t, max_num_io_queues, io_tx_cq_num); 2536 /* 1 IRQ for mgmnt and 1 IRQ for each TX/RX pair */ 2537 max_num_io_queues = min_t(uint32_t, max_num_io_queues, 2538 pci_msix_count(pdev) - 1); 2539 #ifdef RSS 2540 max_num_io_queues = min_t(uint32_t, max_num_io_queues, 2541 rss_getnumbuckets()); 2542 #endif 2543 2544 return (max_num_io_queues); 2545 } 2546 2547 static int 2548 ena_enable_wc(device_t pdev, struct resource *res) 2549 { 2550 #if defined(__i386) || defined(__amd64) || defined(__aarch64__) 2551 vm_offset_t va; 2552 vm_size_t len; 2553 int rc; 2554 2555 va = (vm_offset_t)rman_get_virtual(res); 2556 len = rman_get_size(res); 2557 /* Enable write combining */ 2558 rc = pmap_change_attr(va, len, VM_MEMATTR_WRITE_COMBINING); 2559 if (unlikely(rc != 0)) { 2560 ena_log(pdev, ERR, "pmap_change_attr failed, %d\n", rc); 2561 return (rc); 2562 } 2563 2564 return (0); 2565 #endif 2566 return (EOPNOTSUPP); 2567 } 2568 2569 static int 2570 ena_set_queues_placement_policy(device_t pdev, struct ena_com_dev *ena_dev, 2571 struct ena_admin_feature_llq_desc *llq, 2572 struct ena_llq_configurations *llq_default_configurations) 2573 { 2574 struct ena_adapter *adapter = device_get_softc(pdev); 2575 int rc, rid; 2576 uint32_t llq_feature_mask; 2577 2578 llq_feature_mask = 1 << ENA_ADMIN_LLQ; 2579 if (!(ena_dev->supported_features & llq_feature_mask)) { 2580 ena_log(pdev, WARN, 2581 "LLQ is not supported. Fallback to host mode policy.\n"); 2582 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2583 return (0); 2584 } 2585 2586 rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations); 2587 if (unlikely(rc != 0)) { 2588 ena_log(pdev, WARN, "Failed to configure the device mode. " 2589 "Fallback to host mode policy.\n"); 2590 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2591 return (0); 2592 } 2593 2594 /* Nothing to config, exit */ 2595 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) 2596 return (0); 2597 2598 /* Try to allocate resources for LLQ bar */ 2599 rid = PCIR_BAR(ENA_MEM_BAR); 2600 adapter->memory = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 2601 &rid, RF_ACTIVE); 2602 if (unlikely(adapter->memory == NULL)) { 2603 ena_log(pdev, WARN, "unable to allocate LLQ bar resource. " 2604 "Fallback to host mode policy.\n"); 2605 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 2606 return (0); 2607 } 2608 2609 /* Enable write combining for better LLQ performance */ 2610 rc = ena_enable_wc(adapter->pdev, adapter->memory); 2611 if (unlikely(rc != 0)) { 2612 ena_log(pdev, ERR, "failed to enable write combining.\n"); 2613 return (rc); 2614 } 2615 2616 /* 2617 * Save virtual address of the device's memory region 2618 * for the ena_com layer. 2619 */ 2620 ena_dev->mem_bar = rman_get_virtual(adapter->memory); 2621 2622 return (0); 2623 } 2624 2625 static inline 2626 void set_default_llq_configurations(struct ena_llq_configurations *llq_config, 2627 struct ena_admin_feature_llq_desc *llq) 2628 { 2629 2630 llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER; 2631 llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY; 2632 llq_config->llq_num_decs_before_header = 2633 ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2; 2634 if ((llq->entry_size_ctrl_supported & 2635 ENA_ADMIN_LIST_ENTRY_SIZE_256B) != 0 && 2636 ena_force_large_llq_header) { 2637 llq_config->llq_ring_entry_size = 2638 ENA_ADMIN_LIST_ENTRY_SIZE_256B; 2639 llq_config->llq_ring_entry_size_value = 256; 2640 } else { 2641 llq_config->llq_ring_entry_size = 2642 ENA_ADMIN_LIST_ENTRY_SIZE_128B; 2643 llq_config->llq_ring_entry_size_value = 128; 2644 } 2645 } 2646 2647 static int 2648 ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *ctx) 2649 { 2650 struct ena_admin_feature_llq_desc *llq = &ctx->get_feat_ctx->llq; 2651 struct ena_com_dev *ena_dev = ctx->ena_dev; 2652 uint32_t tx_queue_size = ENA_DEFAULT_RING_SIZE; 2653 uint32_t rx_queue_size = ENA_DEFAULT_RING_SIZE; 2654 uint32_t max_tx_queue_size; 2655 uint32_t max_rx_queue_size; 2656 2657 if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) { 2658 struct ena_admin_queue_ext_feature_fields *max_queue_ext = 2659 &ctx->get_feat_ctx->max_queue_ext.max_queue_ext; 2660 max_rx_queue_size = min_t(uint32_t, 2661 max_queue_ext->max_rx_cq_depth, 2662 max_queue_ext->max_rx_sq_depth); 2663 max_tx_queue_size = max_queue_ext->max_tx_cq_depth; 2664 2665 if (ena_dev->tx_mem_queue_type == 2666 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2667 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2668 llq->max_llq_depth); 2669 else 2670 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2671 max_queue_ext->max_tx_sq_depth); 2672 2673 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2674 max_queue_ext->max_per_packet_tx_descs); 2675 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2676 max_queue_ext->max_per_packet_rx_descs); 2677 } else { 2678 struct ena_admin_queue_feature_desc *max_queues = 2679 &ctx->get_feat_ctx->max_queues; 2680 max_rx_queue_size = min_t(uint32_t, 2681 max_queues->max_cq_depth, 2682 max_queues->max_sq_depth); 2683 max_tx_queue_size = max_queues->max_cq_depth; 2684 2685 if (ena_dev->tx_mem_queue_type == 2686 ENA_ADMIN_PLACEMENT_POLICY_DEV) 2687 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2688 llq->max_llq_depth); 2689 else 2690 max_tx_queue_size = min_t(uint32_t, max_tx_queue_size, 2691 max_queues->max_sq_depth); 2692 2693 ctx->max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2694 max_queues->max_packet_tx_descs); 2695 ctx->max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS, 2696 max_queues->max_packet_rx_descs); 2697 } 2698 2699 /* round down to the nearest power of 2 */ 2700 max_tx_queue_size = 1 << (flsl(max_tx_queue_size) - 1); 2701 max_rx_queue_size = 1 << (flsl(max_rx_queue_size) - 1); 2702 2703 /* 2704 * When forcing large headers, we multiply the entry size by 2, 2705 * and therefore divide the queue size by 2, leaving the amount 2706 * of memory used by the queues unchanged. 2707 */ 2708 if (ena_force_large_llq_header) { 2709 if ((llq->entry_size_ctrl_supported & 2710 ENA_ADMIN_LIST_ENTRY_SIZE_256B) != 0 && 2711 ena_dev->tx_mem_queue_type == 2712 ENA_ADMIN_PLACEMENT_POLICY_DEV) { 2713 max_tx_queue_size /= 2; 2714 ena_log(ctx->pdev, INFO, 2715 "Forcing large headers and decreasing maximum Tx queue size to %d\n", 2716 max_tx_queue_size); 2717 } else { 2718 ena_log(ctx->pdev, WARN, 2719 "Forcing large headers failed: LLQ is disabled or device does not support large headers\n"); 2720 } 2721 } 2722 2723 tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE, 2724 max_tx_queue_size); 2725 rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE, 2726 max_rx_queue_size); 2727 2728 tx_queue_size = 1 << (flsl(tx_queue_size) - 1); 2729 rx_queue_size = 1 << (flsl(rx_queue_size) - 1); 2730 2731 ctx->max_tx_queue_size = max_tx_queue_size; 2732 ctx->max_rx_queue_size = max_rx_queue_size; 2733 ctx->tx_queue_size = tx_queue_size; 2734 ctx->rx_queue_size = rx_queue_size; 2735 2736 return (0); 2737 } 2738 2739 static void 2740 ena_config_host_info(struct ena_com_dev *ena_dev, device_t dev) 2741 { 2742 struct ena_admin_host_info *host_info; 2743 uintptr_t rid; 2744 int rc; 2745 2746 /* Allocate only the host info */ 2747 rc = ena_com_allocate_host_info(ena_dev); 2748 if (unlikely(rc != 0)) { 2749 ena_log(dev, ERR, "Cannot allocate host info\n"); 2750 return; 2751 } 2752 2753 host_info = ena_dev->host_attr.host_info; 2754 2755 if (pci_get_id(dev, PCI_ID_RID, &rid) == 0) 2756 host_info->bdf = rid; 2757 host_info->os_type = ENA_ADMIN_OS_FREEBSD; 2758 host_info->kernel_ver = osreldate; 2759 2760 sprintf(host_info->kernel_ver_str, "%d", osreldate); 2761 host_info->os_dist = 0; 2762 strncpy(host_info->os_dist_str, osrelease, 2763 sizeof(host_info->os_dist_str) - 1); 2764 2765 host_info->driver_version = 2766 (DRV_MODULE_VER_MAJOR) | 2767 (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | 2768 (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT); 2769 host_info->num_cpus = mp_ncpus; 2770 host_info->driver_supported_features = 2771 ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK | 2772 ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK; 2773 2774 rc = ena_com_set_host_attributes(ena_dev); 2775 if (unlikely(rc != 0)) { 2776 if (rc == EOPNOTSUPP) 2777 ena_log(dev, WARN, "Cannot set host attributes\n"); 2778 else 2779 ena_log(dev, ERR, "Cannot set host attributes\n"); 2780 2781 goto err; 2782 } 2783 2784 return; 2785 2786 err: 2787 ena_com_delete_host_info(ena_dev); 2788 } 2789 2790 static int 2791 ena_device_init(struct ena_adapter *adapter, device_t pdev, 2792 struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active) 2793 { 2794 struct ena_com_dev* ena_dev = adapter->ena_dev; 2795 bool readless_supported; 2796 uint32_t aenq_groups; 2797 int dma_width; 2798 int rc; 2799 2800 rc = ena_com_mmio_reg_read_request_init(ena_dev); 2801 if (unlikely(rc != 0)) { 2802 ena_log(pdev, ERR, "failed to init mmio read less\n"); 2803 return (rc); 2804 } 2805 2806 /* 2807 * The PCIe configuration space revision id indicate if mmio reg 2808 * read is disabled 2809 */ 2810 readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ); 2811 ena_com_set_mmio_read_mode(ena_dev, readless_supported); 2812 2813 rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL); 2814 if (unlikely(rc != 0)) { 2815 ena_log(pdev, ERR, "Can not reset device\n"); 2816 goto err_mmio_read_less; 2817 } 2818 2819 rc = ena_com_validate_version(ena_dev); 2820 if (unlikely(rc != 0)) { 2821 ena_log(pdev, ERR, "device version is too low\n"); 2822 goto err_mmio_read_less; 2823 } 2824 2825 dma_width = ena_com_get_dma_width(ena_dev); 2826 if (unlikely(dma_width < 0)) { 2827 ena_log(pdev, ERR, "Invalid dma width value %d", dma_width); 2828 rc = dma_width; 2829 goto err_mmio_read_less; 2830 } 2831 adapter->dma_width = dma_width; 2832 2833 /* ENA admin level init */ 2834 rc = ena_com_admin_init(ena_dev, &aenq_handlers); 2835 if (unlikely(rc != 0)) { 2836 ena_log(pdev, ERR, 2837 "Can not initialize ena admin queue with device\n"); 2838 goto err_mmio_read_less; 2839 } 2840 2841 /* 2842 * To enable the msix interrupts the driver needs to know the number 2843 * of queues. So the driver uses polling mode to retrieve this 2844 * information 2845 */ 2846 ena_com_set_admin_polling_mode(ena_dev, true); 2847 2848 ena_config_host_info(ena_dev, pdev); 2849 2850 /* Get Device Attributes */ 2851 rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx); 2852 if (unlikely(rc != 0)) { 2853 ena_log(pdev, ERR, 2854 "Cannot get attribute for ena device rc: %d\n", rc); 2855 goto err_admin_init; 2856 } 2857 2858 aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | 2859 BIT(ENA_ADMIN_FATAL_ERROR) | 2860 BIT(ENA_ADMIN_WARNING) | 2861 BIT(ENA_ADMIN_NOTIFICATION) | 2862 BIT(ENA_ADMIN_KEEP_ALIVE); 2863 2864 aenq_groups &= get_feat_ctx->aenq.supported_groups; 2865 rc = ena_com_set_aenq_config(ena_dev, aenq_groups); 2866 if (unlikely(rc != 0)) { 2867 ena_log(pdev, ERR, "Cannot configure aenq groups rc: %d\n", rc); 2868 goto err_admin_init; 2869 } 2870 2871 *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE)); 2872 2873 return (0); 2874 2875 err_admin_init: 2876 ena_com_delete_host_info(ena_dev); 2877 ena_com_admin_destroy(ena_dev); 2878 err_mmio_read_less: 2879 ena_com_mmio_reg_read_request_destroy(ena_dev); 2880 2881 return (rc); 2882 } 2883 2884 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter) 2885 { 2886 struct ena_com_dev *ena_dev = adapter->ena_dev; 2887 int rc; 2888 2889 rc = ena_enable_msix(adapter); 2890 if (unlikely(rc != 0)) { 2891 ena_log(adapter->pdev, ERR, "Error with MSI-X enablement\n"); 2892 return (rc); 2893 } 2894 2895 ena_setup_mgmnt_intr(adapter); 2896 2897 rc = ena_request_mgmnt_irq(adapter); 2898 if (unlikely(rc != 0)) { 2899 ena_log(adapter->pdev, ERR, "Cannot setup mgmnt queue intr\n"); 2900 goto err_disable_msix; 2901 } 2902 2903 ena_com_set_admin_polling_mode(ena_dev, false); 2904 2905 ena_com_admin_aenq_enable(ena_dev); 2906 2907 return (0); 2908 2909 err_disable_msix: 2910 ena_disable_msix(adapter); 2911 2912 return (rc); 2913 } 2914 2915 /* Function called on ENA_ADMIN_KEEP_ALIVE event */ 2916 static void ena_keep_alive_wd(void *adapter_data, 2917 struct ena_admin_aenq_entry *aenq_e) 2918 { 2919 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 2920 struct ena_admin_aenq_keep_alive_desc *desc; 2921 sbintime_t stime; 2922 uint64_t rx_drops; 2923 uint64_t tx_drops; 2924 2925 desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e; 2926 2927 rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low; 2928 tx_drops = ((uint64_t)desc->tx_drops_high << 32) | desc->tx_drops_low; 2929 counter_u64_zero(adapter->hw_stats.rx_drops); 2930 counter_u64_add(adapter->hw_stats.rx_drops, rx_drops); 2931 counter_u64_zero(adapter->hw_stats.tx_drops); 2932 counter_u64_add(adapter->hw_stats.tx_drops, tx_drops); 2933 2934 stime = getsbinuptime(); 2935 atomic_store_rel_64(&adapter->keep_alive_timestamp, stime); 2936 } 2937 2938 /* Check for keep alive expiration */ 2939 static void check_for_missing_keep_alive(struct ena_adapter *adapter) 2940 { 2941 sbintime_t timestamp, time; 2942 2943 if (adapter->wd_active == 0) 2944 return; 2945 2946 if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT) 2947 return; 2948 2949 timestamp = atomic_load_acq_64(&adapter->keep_alive_timestamp); 2950 time = getsbinuptime() - timestamp; 2951 if (unlikely(time > adapter->keep_alive_timeout)) { 2952 ena_log(adapter->pdev, ERR, "Keep alive watchdog timeout.\n"); 2953 counter_u64_add(adapter->dev_stats.wd_expired, 1); 2954 ena_trigger_reset(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO); 2955 } 2956 } 2957 2958 /* Check if admin queue is enabled */ 2959 static void check_for_admin_com_state(struct ena_adapter *adapter) 2960 { 2961 if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) == 2962 false)) { 2963 ena_log(adapter->pdev, ERR, 2964 "ENA admin queue is not in running state!\n"); 2965 counter_u64_add(adapter->dev_stats.admin_q_pause, 1); 2966 ena_trigger_reset(adapter, ENA_REGS_RESET_ADMIN_TO); 2967 } 2968 } 2969 2970 static int 2971 check_for_rx_interrupt_queue(struct ena_adapter *adapter, 2972 struct ena_ring *rx_ring) 2973 { 2974 if (likely(rx_ring->first_interrupt)) 2975 return (0); 2976 2977 if (ena_com_cq_empty(rx_ring->ena_com_io_cq)) 2978 return (0); 2979 2980 rx_ring->no_interrupt_event_cnt++; 2981 2982 if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) { 2983 ena_log(adapter->pdev, ERR, "Potential MSIX issue on Rx side " 2984 "Queue = %d. Reset the device\n", rx_ring->qid); 2985 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_INTERRUPT); 2986 return (EIO); 2987 } 2988 2989 return (0); 2990 } 2991 2992 static int 2993 check_missing_comp_in_tx_queue(struct ena_adapter *adapter, 2994 struct ena_ring *tx_ring) 2995 { 2996 device_t pdev = adapter->pdev; 2997 struct bintime curtime, time; 2998 struct ena_tx_buffer *tx_buf; 2999 sbintime_t time_offset; 3000 uint32_t missed_tx = 0; 3001 int i, rc = 0; 3002 3003 getbinuptime(&curtime); 3004 3005 for (i = 0; i < tx_ring->ring_size; i++) { 3006 tx_buf = &tx_ring->tx_buffer_info[i]; 3007 3008 if (bintime_isset(&tx_buf->timestamp) == 0) 3009 continue; 3010 3011 time = curtime; 3012 bintime_sub(&time, &tx_buf->timestamp); 3013 time_offset = bttosbt(time); 3014 3015 if (unlikely(!tx_ring->first_interrupt && 3016 time_offset > 2 * adapter->missing_tx_timeout)) { 3017 /* 3018 * If after graceful period interrupt is still not 3019 * received, we schedule a reset. 3020 */ 3021 ena_log(pdev, ERR, 3022 "Potential MSIX issue on Tx side Queue = %d. " 3023 "Reset the device\n", tx_ring->qid); 3024 ena_trigger_reset(adapter, 3025 ENA_REGS_RESET_MISS_INTERRUPT); 3026 return (EIO); 3027 } 3028 3029 /* Check again if packet is still waiting */ 3030 if (unlikely(time_offset > adapter->missing_tx_timeout)) { 3031 3032 if (!tx_buf->print_once) 3033 ena_log(pdev, WARN, "Found a Tx that wasn't " 3034 "completed on time, qid %d, index %d.\n", 3035 tx_ring->qid, i); 3036 3037 tx_buf->print_once = true; 3038 missed_tx++; 3039 } 3040 } 3041 3042 if (unlikely(missed_tx > adapter->missing_tx_threshold)) { 3043 ena_log(pdev, ERR, 3044 "The number of lost tx completion is above the threshold " 3045 "(%d > %d). Reset the device\n", 3046 missed_tx, adapter->missing_tx_threshold); 3047 ena_trigger_reset(adapter, ENA_REGS_RESET_MISS_TX_CMPL); 3048 rc = EIO; 3049 } 3050 3051 counter_u64_add(tx_ring->tx_stats.missing_tx_comp, missed_tx); 3052 3053 return (rc); 3054 } 3055 3056 /* 3057 * Check for TX which were not completed on time. 3058 * Timeout is defined by "missing_tx_timeout". 3059 * Reset will be performed if number of incompleted 3060 * transactions exceeds "missing_tx_threshold". 3061 */ 3062 static void 3063 check_for_missing_completions(struct ena_adapter *adapter) 3064 { 3065 struct ena_ring *tx_ring; 3066 struct ena_ring *rx_ring; 3067 int i, budget, rc; 3068 3069 /* Make sure the driver doesn't turn the device in other process */ 3070 rmb(); 3071 3072 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3073 return; 3074 3075 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3076 return; 3077 3078 if (adapter->missing_tx_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3079 return; 3080 3081 budget = adapter->missing_tx_max_queues; 3082 3083 for (i = adapter->next_monitored_tx_qid; i < adapter->num_io_queues; i++) { 3084 tx_ring = &adapter->tx_ring[i]; 3085 rx_ring = &adapter->rx_ring[i]; 3086 3087 rc = check_missing_comp_in_tx_queue(adapter, tx_ring); 3088 if (unlikely(rc != 0)) 3089 return; 3090 3091 rc = check_for_rx_interrupt_queue(adapter, rx_ring); 3092 if (unlikely(rc != 0)) 3093 return; 3094 3095 budget--; 3096 if (budget == 0) { 3097 i++; 3098 break; 3099 } 3100 } 3101 3102 adapter->next_monitored_tx_qid = i % adapter->num_io_queues; 3103 } 3104 3105 /* trigger rx cleanup after 2 consecutive detections */ 3106 #define EMPTY_RX_REFILL 2 3107 /* For the rare case where the device runs out of Rx descriptors and the 3108 * msix handler failed to refill new Rx descriptors (due to a lack of memory 3109 * for example). 3110 * This case will lead to a deadlock: 3111 * The device won't send interrupts since all the new Rx packets will be dropped 3112 * The msix handler won't allocate new Rx descriptors so the device won't be 3113 * able to send new packets. 3114 * 3115 * When such a situation is detected - execute rx cleanup task in another thread 3116 */ 3117 static void 3118 check_for_empty_rx_ring(struct ena_adapter *adapter) 3119 { 3120 struct ena_ring *rx_ring; 3121 int i, refill_required; 3122 3123 if (!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3124 return; 3125 3126 if (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter)) 3127 return; 3128 3129 for (i = 0; i < adapter->num_io_queues; i++) { 3130 rx_ring = &adapter->rx_ring[i]; 3131 3132 refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq); 3133 if (unlikely(refill_required == (rx_ring->ring_size - 1))) { 3134 rx_ring->empty_rx_queue++; 3135 3136 if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) { 3137 counter_u64_add(rx_ring->rx_stats.empty_rx_ring, 3138 1); 3139 3140 ena_log(adapter->pdev, WARN, 3141 "Rx ring %d is stalled. Triggering the refill function\n", 3142 i); 3143 3144 taskqueue_enqueue(rx_ring->que->cleanup_tq, 3145 &rx_ring->que->cleanup_task); 3146 rx_ring->empty_rx_queue = 0; 3147 } 3148 } else { 3149 rx_ring->empty_rx_queue = 0; 3150 } 3151 } 3152 } 3153 3154 static void ena_update_hints(struct ena_adapter *adapter, 3155 struct ena_admin_ena_hw_hints *hints) 3156 { 3157 struct ena_com_dev *ena_dev = adapter->ena_dev; 3158 3159 if (hints->admin_completion_tx_timeout) 3160 ena_dev->admin_queue.completion_timeout = 3161 hints->admin_completion_tx_timeout * 1000; 3162 3163 if (hints->mmio_read_timeout) 3164 /* convert to usec */ 3165 ena_dev->mmio_read.reg_read_to = 3166 hints->mmio_read_timeout * 1000; 3167 3168 if (hints->missed_tx_completion_count_threshold_to_reset) 3169 adapter->missing_tx_threshold = 3170 hints->missed_tx_completion_count_threshold_to_reset; 3171 3172 if (hints->missing_tx_completion_timeout) { 3173 if (hints->missing_tx_completion_timeout == 3174 ENA_HW_HINTS_NO_TIMEOUT) 3175 adapter->missing_tx_timeout = ENA_HW_HINTS_NO_TIMEOUT; 3176 else 3177 adapter->missing_tx_timeout = 3178 SBT_1MS * hints->missing_tx_completion_timeout; 3179 } 3180 3181 if (hints->driver_watchdog_timeout) { 3182 if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT) 3183 adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT; 3184 else 3185 adapter->keep_alive_timeout = 3186 SBT_1MS * hints->driver_watchdog_timeout; 3187 } 3188 } 3189 3190 /** 3191 * ena_copy_eni_metrics - Get and copy ENI metrics from the HW. 3192 * @adapter: ENA device adapter 3193 * 3194 * Returns 0 on success, EOPNOTSUPP if current HW doesn't support those metrics 3195 * and other error codes on failure. 3196 * 3197 * This function can possibly cause a race with other calls to the admin queue. 3198 * Because of that, the caller should either lock this function or make sure 3199 * that there is no race in the current context. 3200 */ 3201 static int 3202 ena_copy_eni_metrics(struct ena_adapter *adapter) 3203 { 3204 static bool print_once = true; 3205 int rc; 3206 3207 rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_metrics); 3208 3209 if (rc != 0) { 3210 if (rc == ENA_COM_UNSUPPORTED) { 3211 if (print_once) { 3212 ena_log(adapter->pdev, WARN, 3213 "Retrieving ENI metrics is not supported.\n"); 3214 print_once = false; 3215 } else { 3216 ena_log(adapter->pdev, DBG, 3217 "Retrieving ENI metrics is not supported.\n"); 3218 } 3219 } else { 3220 ena_log(adapter->pdev, ERR, 3221 "Failed to get ENI metrics: %d\n", rc); 3222 } 3223 } 3224 3225 return (rc); 3226 } 3227 3228 static void 3229 ena_timer_service(void *data) 3230 { 3231 struct ena_adapter *adapter = (struct ena_adapter *)data; 3232 struct ena_admin_host_info *host_info = 3233 adapter->ena_dev->host_attr.host_info; 3234 3235 check_for_missing_keep_alive(adapter); 3236 3237 check_for_admin_com_state(adapter); 3238 3239 check_for_missing_completions(adapter); 3240 3241 check_for_empty_rx_ring(adapter); 3242 3243 /* 3244 * User controller update of the ENI metrics. 3245 * If the delay was set to 0, then the stats shouldn't be updated at 3246 * all. 3247 * Otherwise, wait 'eni_metrics_sample_interval' seconds, before 3248 * updating stats. 3249 * As timer service is executed every second, it's enough to increment 3250 * appropriate counter each time the timer service is executed. 3251 */ 3252 if ((adapter->eni_metrics_sample_interval != 0) && 3253 (++adapter->eni_metrics_sample_interval_cnt >= 3254 adapter->eni_metrics_sample_interval)) { 3255 /* 3256 * There is no race with other admin queue calls, as: 3257 * - Timer service runs after attach function ends, so all 3258 * configuration calls to the admin queue are finished. 3259 * - Timer service is temporarily stopped when bringing 3260 * the interface up or down. 3261 * - After interface is up, the driver doesn't use (at least 3262 * for now) other functions writing to the admin queue. 3263 * 3264 * It may change in the future, so in that situation, the lock 3265 * will be needed. ENA_LOCK_*() cannot be used for that purpose, 3266 * as callout ena_timer_service is protected by them. It could 3267 * lead to the deadlock if callout_drain() would hold the lock 3268 * before ena_copy_eni_metrics() was executed. It's advised to 3269 * use separate lock in that situation which will be used only 3270 * for the admin queue. 3271 */ 3272 (void)ena_copy_eni_metrics(adapter); 3273 adapter->eni_metrics_sample_interval_cnt = 0; 3274 } 3275 3276 3277 if (host_info != NULL) 3278 ena_update_host_info(host_info, adapter->ifp); 3279 3280 if (unlikely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3281 /* 3282 * Timeout when validating version indicates that the device 3283 * became unresponsive. If that happens skip the reset and 3284 * reschedule timer service, so the reset can be retried later. 3285 */ 3286 if (ena_com_validate_version(adapter->ena_dev) == 3287 ENA_COM_TIMER_EXPIRED) { 3288 ena_log(adapter->pdev, WARN, 3289 "FW unresponsive, skipping reset\n"); 3290 ENA_TIMER_RESET(adapter); 3291 return; 3292 } 3293 ena_log(adapter->pdev, WARN, "Trigger reset is on\n"); 3294 taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task); 3295 return; 3296 } 3297 3298 /* 3299 * Schedule another timeout one second from now. 3300 */ 3301 ENA_TIMER_RESET(adapter); 3302 } 3303 3304 void 3305 ena_destroy_device(struct ena_adapter *adapter, bool graceful) 3306 { 3307 if_t ifp = adapter->ifp; 3308 struct ena_com_dev *ena_dev = adapter->ena_dev; 3309 bool dev_up; 3310 3311 if (!ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) 3312 return; 3313 3314 if_link_state_change(ifp, LINK_STATE_DOWN); 3315 3316 ENA_TIMER_DRAIN(adapter); 3317 3318 dev_up = ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter); 3319 if (dev_up) 3320 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3321 3322 if (!graceful) 3323 ena_com_set_admin_running_state(ena_dev, false); 3324 3325 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, adapter)) 3326 ena_down(adapter); 3327 3328 /* 3329 * Stop the device from sending AENQ events (if the device was up, and 3330 * the trigger reset was on, ena_down already performs device reset) 3331 */ 3332 if (!(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter) && dev_up)) 3333 ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); 3334 3335 ena_free_mgmnt_irq(adapter); 3336 3337 ena_disable_msix(adapter); 3338 3339 /* 3340 * IO rings resources should be freed because `ena_restore_device()` 3341 * calls (not directly) `ena_enable_msix()`, which re-allocates MSIX 3342 * vectors. The amount of MSIX vectors after destroy-restore may be 3343 * different than before. Therefore, IO rings resources should be 3344 * established from scratch each time. 3345 */ 3346 ena_free_all_io_rings_resources(adapter); 3347 3348 ena_com_abort_admin_commands(ena_dev); 3349 3350 ena_com_wait_for_abort_completion(ena_dev); 3351 3352 ena_com_admin_destroy(ena_dev); 3353 3354 ena_com_mmio_reg_read_request_destroy(ena_dev); 3355 3356 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3357 3358 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter); 3359 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3360 } 3361 3362 static int 3363 ena_device_validate_params(struct ena_adapter *adapter, 3364 struct ena_com_dev_get_features_ctx *get_feat_ctx) 3365 { 3366 3367 if (memcmp(get_feat_ctx->dev_attr.mac_addr, adapter->mac_addr, 3368 ETHER_ADDR_LEN) != 0) { 3369 ena_log(adapter->pdev, ERR, "Error, mac addresses differ\n"); 3370 return (EINVAL); 3371 } 3372 3373 if (get_feat_ctx->dev_attr.max_mtu < if_getmtu(adapter->ifp)) { 3374 ena_log(adapter->pdev, ERR, 3375 "Error, device max mtu is smaller than ifp MTU\n"); 3376 return (EINVAL); 3377 } 3378 3379 return 0; 3380 } 3381 3382 int 3383 ena_restore_device(struct ena_adapter *adapter) 3384 { 3385 struct ena_com_dev_get_features_ctx get_feat_ctx; 3386 struct ena_com_dev *ena_dev = adapter->ena_dev; 3387 if_t ifp = adapter->ifp; 3388 device_t dev = adapter->pdev; 3389 int wd_active; 3390 int rc; 3391 3392 ENA_FLAG_SET_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3393 3394 rc = ena_device_init(adapter, dev, &get_feat_ctx, &wd_active); 3395 if (rc != 0) { 3396 ena_log(dev, ERR, "Cannot initialize device\n"); 3397 goto err; 3398 } 3399 /* 3400 * Only enable WD if it was enabled before reset, so it won't override 3401 * value set by the user by the sysctl. 3402 */ 3403 if (adapter->wd_active != 0) 3404 adapter->wd_active = wd_active; 3405 3406 rc = ena_device_validate_params(adapter, &get_feat_ctx); 3407 if (rc != 0) { 3408 ena_log(dev, ERR, "Validation of device parameters failed\n"); 3409 goto err_device_destroy; 3410 } 3411 3412 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3413 /* Make sure we don't have a race with AENQ Links state handler */ 3414 if (ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)) 3415 if_link_state_change(ifp, LINK_STATE_UP); 3416 3417 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 3418 if (rc != 0) { 3419 ena_log(dev, ERR, "Enable MSI-X failed\n"); 3420 goto err_device_destroy; 3421 } 3422 3423 /* 3424 * Effective value of used MSIX vectors should be the same as before 3425 * `ena_destroy_device()`, if possible, or closest to it if less vectors 3426 * are available. 3427 */ 3428 if ((adapter->msix_vecs - ENA_ADMIN_MSIX_VEC) < adapter->num_io_queues) 3429 adapter->num_io_queues = 3430 adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; 3431 3432 /* Re-initialize rings basic information */ 3433 ena_init_io_rings(adapter); 3434 3435 /* If the interface was up before the reset bring it up */ 3436 if (ENA_FLAG_ISSET(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter)) { 3437 rc = ena_up(adapter); 3438 if (rc != 0) { 3439 ena_log(dev, ERR, "Failed to create I/O queues\n"); 3440 goto err_disable_msix; 3441 } 3442 } 3443 3444 /* Indicate that device is running again and ready to work */ 3445 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3446 3447 /* 3448 * As the AENQ handlers weren't executed during reset because 3449 * the flag ENA_FLAG_DEVICE_RUNNING was turned off, the 3450 * timestamp must be updated again That will prevent next reset 3451 * caused by missing keep alive. 3452 */ 3453 adapter->keep_alive_timestamp = getsbinuptime(); 3454 ENA_TIMER_RESET(adapter); 3455 3456 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEV_UP_BEFORE_RESET, adapter); 3457 3458 ena_log(dev, INFO, 3459 "Device reset completed successfully, Driver info: %s\n", ena_version); 3460 3461 return (rc); 3462 3463 err_disable_msix: 3464 ena_free_mgmnt_irq(adapter); 3465 ena_disable_msix(adapter); 3466 err_device_destroy: 3467 ena_com_abort_admin_commands(ena_dev); 3468 ena_com_wait_for_abort_completion(ena_dev); 3469 ena_com_admin_destroy(ena_dev); 3470 ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE); 3471 ena_com_mmio_reg_read_request_destroy(ena_dev); 3472 err: 3473 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3474 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_ONGOING_RESET, adapter); 3475 ena_log(dev, ERR, "Reset attempt failed. Can not reset the device\n"); 3476 3477 ENA_TIMER_RESET(adapter); 3478 3479 return (rc); 3480 } 3481 3482 static void 3483 ena_reset_task(void *arg, int pending) 3484 { 3485 struct ena_adapter *adapter = (struct ena_adapter *)arg; 3486 3487 ENA_LOCK_LOCK(); 3488 if (likely(ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) { 3489 ena_destroy_device(adapter, false); 3490 ena_restore_device(adapter); 3491 } 3492 ENA_LOCK_UNLOCK(); 3493 } 3494 3495 /** 3496 * ena_attach - Device Initialization Routine 3497 * @pdev: device information struct 3498 * 3499 * Returns 0 on success, otherwise on failure. 3500 * 3501 * ena_attach initializes an adapter identified by a device structure. 3502 * The OS initialization, configuring of the adapter private structure, 3503 * and a hardware reset occur. 3504 **/ 3505 static int 3506 ena_attach(device_t pdev) 3507 { 3508 struct ena_com_dev_get_features_ctx get_feat_ctx; 3509 struct ena_llq_configurations llq_config; 3510 struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 }; 3511 static int version_printed; 3512 struct ena_adapter *adapter; 3513 struct ena_com_dev *ena_dev = NULL; 3514 uint32_t max_num_io_queues; 3515 int msix_rid; 3516 int rid, rc; 3517 3518 adapter = device_get_softc(pdev); 3519 adapter->pdev = pdev; 3520 adapter->first_bind = -1; 3521 3522 /* 3523 * Set up the timer service - driver is responsible for avoiding 3524 * concurrency, as the callout won't be using any locking inside. 3525 */ 3526 ENA_TIMER_INIT(adapter); 3527 adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO; 3528 adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO; 3529 adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES; 3530 adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD; 3531 3532 if (version_printed++ == 0) 3533 ena_log(pdev, INFO, "%s\n", ena_version); 3534 3535 /* Allocate memory for ena_dev structure */ 3536 ena_dev = malloc(sizeof(struct ena_com_dev), M_DEVBUF, 3537 M_WAITOK | M_ZERO); 3538 3539 adapter->ena_dev = ena_dev; 3540 ena_dev->dmadev = pdev; 3541 3542 rid = PCIR_BAR(ENA_REG_BAR); 3543 adapter->memory = NULL; 3544 adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 3545 &rid, RF_ACTIVE); 3546 if (unlikely(adapter->registers == NULL)) { 3547 ena_log(pdev, ERR, 3548 "unable to allocate bus resource: registers!\n"); 3549 rc = ENOMEM; 3550 goto err_dev_free; 3551 } 3552 3553 /* MSIx vector table may reside on BAR0 with registers or on BAR1. */ 3554 msix_rid = pci_msix_table_bar(pdev); 3555 if (msix_rid != rid) { 3556 adapter->msix = bus_alloc_resource_any(pdev, SYS_RES_MEMORY, 3557 &msix_rid, RF_ACTIVE); 3558 if (unlikely(adapter->msix == NULL)) { 3559 ena_log(pdev, ERR, 3560 "unable to allocate bus resource: msix!\n"); 3561 rc = ENOMEM; 3562 goto err_pci_free; 3563 } 3564 adapter->msix_rid = msix_rid; 3565 } 3566 3567 ena_dev->bus = malloc(sizeof(struct ena_bus), M_DEVBUF, 3568 M_WAITOK | M_ZERO); 3569 3570 /* Store register resources */ 3571 ((struct ena_bus*)(ena_dev->bus))->reg_bar_t = 3572 rman_get_bustag(adapter->registers); 3573 ((struct ena_bus*)(ena_dev->bus))->reg_bar_h = 3574 rman_get_bushandle(adapter->registers); 3575 3576 if (unlikely(((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0)) { 3577 ena_log(pdev, ERR, "failed to pmap registers bar\n"); 3578 rc = ENXIO; 3579 goto err_bus_free; 3580 } 3581 3582 ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST; 3583 3584 /* Initially clear all the flags */ 3585 ENA_FLAG_ZERO(adapter); 3586 3587 /* Device initialization */ 3588 rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active); 3589 if (unlikely(rc != 0)) { 3590 ena_log(pdev, ERR, "ENA device init failed! (err: %d)\n", rc); 3591 rc = ENXIO; 3592 goto err_bus_free; 3593 } 3594 3595 set_default_llq_configurations(&llq_config, &get_feat_ctx.llq); 3596 3597 rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx.llq, 3598 &llq_config); 3599 if (unlikely(rc != 0)) { 3600 ena_log(pdev, ERR, "failed to set placement policy\n"); 3601 goto err_com_free; 3602 } 3603 3604 if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) 3605 adapter->disable_meta_caching = 3606 !!(get_feat_ctx.llq.accel_mode.u.get.supported_flags & 3607 BIT(ENA_ADMIN_DISABLE_META_CACHING)); 3608 3609 adapter->keep_alive_timestamp = getsbinuptime(); 3610 3611 adapter->tx_offload_cap = get_feat_ctx.offload.tx; 3612 3613 memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr, 3614 ETHER_ADDR_LEN); 3615 3616 calc_queue_ctx.pdev = pdev; 3617 calc_queue_ctx.ena_dev = ena_dev; 3618 calc_queue_ctx.get_feat_ctx = &get_feat_ctx; 3619 3620 /* Calculate initial and maximum IO queue number and size */ 3621 max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, 3622 &get_feat_ctx); 3623 rc = ena_calc_io_queue_size(&calc_queue_ctx); 3624 if (unlikely((rc != 0) || (max_num_io_queues <= 0))) { 3625 rc = EFAULT; 3626 goto err_com_free; 3627 } 3628 3629 adapter->requested_tx_ring_size = calc_queue_ctx.tx_queue_size; 3630 adapter->requested_rx_ring_size = calc_queue_ctx.rx_queue_size; 3631 adapter->max_tx_ring_size = calc_queue_ctx.max_tx_queue_size; 3632 adapter->max_rx_ring_size = calc_queue_ctx.max_rx_queue_size; 3633 adapter->max_tx_sgl_size = calc_queue_ctx.max_tx_sgl_size; 3634 adapter->max_rx_sgl_size = calc_queue_ctx.max_rx_sgl_size; 3635 3636 adapter->max_num_io_queues = max_num_io_queues; 3637 3638 adapter->buf_ring_size = ENA_DEFAULT_BUF_RING_SIZE; 3639 3640 adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu; 3641 3642 adapter->reset_reason = ENA_REGS_RESET_NORMAL; 3643 3644 /* set up dma tags for rx and tx buffers */ 3645 rc = ena_setup_tx_dma_tag(adapter); 3646 if (unlikely(rc != 0)) { 3647 ena_log(pdev, ERR, "Failed to create TX DMA tag\n"); 3648 goto err_com_free; 3649 } 3650 3651 rc = ena_setup_rx_dma_tag(adapter); 3652 if (unlikely(rc != 0)) { 3653 ena_log(pdev, ERR, "Failed to create RX DMA tag\n"); 3654 goto err_tx_tag_free; 3655 } 3656 3657 /* 3658 * The amount of requested MSIX vectors is equal to 3659 * adapter::max_num_io_queues (see `ena_enable_msix()`), plus a constant 3660 * number of admin queue interrupts. The former is initially determined 3661 * by HW capabilities (see `ena_calc_max_io_queue_num())` but may not be 3662 * achieved if there are not enough system resources. By default, the 3663 * number of effectively used IO queues is the same but later on it can 3664 * be limited by the user using sysctl interface. 3665 */ 3666 rc = ena_enable_msix_and_set_admin_interrupts(adapter); 3667 if (unlikely(rc != 0)) { 3668 ena_log(pdev, ERR, 3669 "Failed to enable and set the admin interrupts\n"); 3670 goto err_io_free; 3671 } 3672 /* By default all of allocated MSIX vectors are actively used */ 3673 adapter->num_io_queues = adapter->msix_vecs - ENA_ADMIN_MSIX_VEC; 3674 3675 /* initialize rings basic information */ 3676 ena_init_io_rings(adapter); 3677 3678 /* setup network interface */ 3679 rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx); 3680 if (unlikely(rc != 0)) { 3681 ena_log(pdev, ERR, "Error with network interface setup\n"); 3682 goto err_msix_free; 3683 } 3684 3685 /* Initialize reset task queue */ 3686 TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter); 3687 adapter->reset_tq = taskqueue_create("ena_reset_enqueue", 3688 M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq); 3689 taskqueue_start_threads(&adapter->reset_tq, 1, PI_NET, 3690 "%s rstq", device_get_nameunit(adapter->pdev)); 3691 3692 /* Initialize statistics */ 3693 ena_alloc_counters((counter_u64_t *)&adapter->dev_stats, 3694 sizeof(struct ena_stats_dev)); 3695 ena_alloc_counters((counter_u64_t *)&adapter->hw_stats, 3696 sizeof(struct ena_hw_stats)); 3697 ena_sysctl_add_nodes(adapter); 3698 3699 #ifdef DEV_NETMAP 3700 rc = ena_netmap_attach(adapter); 3701 if (rc != 0) { 3702 ena_log(pdev, ERR, "netmap attach failed: %d\n", rc); 3703 goto err_detach; 3704 } 3705 #endif /* DEV_NETMAP */ 3706 3707 /* Tell the stack that the interface is not active */ 3708 if_setdrvflagbits(adapter->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING); 3709 ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEVICE_RUNNING, adapter); 3710 3711 /* Run the timer service */ 3712 ENA_TIMER_RESET(adapter); 3713 3714 return (0); 3715 3716 #ifdef DEV_NETMAP 3717 err_detach: 3718 ether_ifdetach(adapter->ifp); 3719 #endif /* DEV_NETMAP */ 3720 err_msix_free: 3721 ena_com_dev_reset(adapter->ena_dev, ENA_REGS_RESET_INIT_ERR); 3722 ena_free_mgmnt_irq(adapter); 3723 ena_disable_msix(adapter); 3724 err_io_free: 3725 ena_free_all_io_rings_resources(adapter); 3726 ena_free_rx_dma_tag(adapter); 3727 err_tx_tag_free: 3728 ena_free_tx_dma_tag(adapter); 3729 err_com_free: 3730 ena_com_admin_destroy(ena_dev); 3731 ena_com_delete_host_info(ena_dev); 3732 ena_com_mmio_reg_read_request_destroy(ena_dev); 3733 err_bus_free: 3734 free(ena_dev->bus, M_DEVBUF); 3735 err_pci_free: 3736 ena_free_pci_resources(adapter); 3737 err_dev_free: 3738 free(ena_dev, M_DEVBUF); 3739 3740 return (rc); 3741 } 3742 3743 /** 3744 * ena_detach - Device Removal Routine 3745 * @pdev: device information struct 3746 * 3747 * ena_detach is called by the device subsystem to alert the driver 3748 * that it should release a PCI device. 3749 **/ 3750 static int 3751 ena_detach(device_t pdev) 3752 { 3753 struct ena_adapter *adapter = device_get_softc(pdev); 3754 struct ena_com_dev *ena_dev = adapter->ena_dev; 3755 int rc; 3756 3757 /* Make sure VLANS are not using driver */ 3758 if (adapter->ifp->if_vlantrunk != NULL) { 3759 ena_log(adapter->pdev, ERR, "VLAN is in use, detach first\n"); 3760 return (EBUSY); 3761 } 3762 3763 ether_ifdetach(adapter->ifp); 3764 3765 /* Stop timer service */ 3766 ENA_LOCK_LOCK(); 3767 ENA_TIMER_DRAIN(adapter); 3768 ENA_LOCK_UNLOCK(); 3769 3770 /* Release reset task */ 3771 while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL)) 3772 taskqueue_drain(adapter->reset_tq, &adapter->reset_task); 3773 taskqueue_free(adapter->reset_tq); 3774 3775 ENA_LOCK_LOCK(); 3776 ena_down(adapter); 3777 ena_destroy_device(adapter, true); 3778 ENA_LOCK_UNLOCK(); 3779 3780 /* Restore unregistered sysctl queue nodes. */ 3781 ena_sysctl_update_queue_node_nb(adapter, adapter->num_io_queues, 3782 adapter->max_num_io_queues); 3783 3784 #ifdef DEV_NETMAP 3785 netmap_detach(adapter->ifp); 3786 #endif /* DEV_NETMAP */ 3787 3788 ena_free_counters((counter_u64_t *)&adapter->hw_stats, 3789 sizeof(struct ena_hw_stats)); 3790 ena_free_counters((counter_u64_t *)&adapter->dev_stats, 3791 sizeof(struct ena_stats_dev)); 3792 3793 rc = ena_free_rx_dma_tag(adapter); 3794 if (unlikely(rc != 0)) 3795 ena_log(adapter->pdev, WARN, 3796 "Unmapped RX DMA tag associations\n"); 3797 3798 rc = ena_free_tx_dma_tag(adapter); 3799 if (unlikely(rc != 0)) 3800 ena_log(adapter->pdev, WARN, 3801 "Unmapped TX DMA tag associations\n"); 3802 3803 ena_free_irqs(adapter); 3804 3805 ena_free_pci_resources(adapter); 3806 3807 if (adapter->rss_indir != NULL) 3808 free(adapter->rss_indir, M_DEVBUF); 3809 3810 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) 3811 ena_com_rss_destroy(ena_dev); 3812 3813 ena_com_delete_host_info(ena_dev); 3814 3815 if_free(adapter->ifp); 3816 3817 free(ena_dev->bus, M_DEVBUF); 3818 3819 free(ena_dev, M_DEVBUF); 3820 3821 return (bus_generic_detach(pdev)); 3822 } 3823 3824 /****************************************************************************** 3825 ******************************** AENQ Handlers ******************************* 3826 *****************************************************************************/ 3827 /** 3828 * ena_update_on_link_change: 3829 * Notify the network interface about the change in link status 3830 **/ 3831 static void 3832 ena_update_on_link_change(void *adapter_data, 3833 struct ena_admin_aenq_entry *aenq_e) 3834 { 3835 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3836 struct ena_admin_aenq_link_change_desc *aenq_desc; 3837 int status; 3838 if_t ifp; 3839 3840 aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e; 3841 ifp = adapter->ifp; 3842 status = aenq_desc->flags & 3843 ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK; 3844 3845 if (status != 0) { 3846 ena_log(adapter->pdev, INFO, "link is UP\n"); 3847 ENA_FLAG_SET_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3848 if (!ENA_FLAG_ISSET(ENA_FLAG_ONGOING_RESET, adapter)) 3849 if_link_state_change(ifp, LINK_STATE_UP); 3850 } else { 3851 ena_log(adapter->pdev, INFO, "link is DOWN\n"); 3852 if_link_state_change(ifp, LINK_STATE_DOWN); 3853 ENA_FLAG_CLEAR_ATOMIC(ENA_FLAG_LINK_UP, adapter); 3854 } 3855 } 3856 3857 static void ena_notification(void *adapter_data, 3858 struct ena_admin_aenq_entry *aenq_e) 3859 { 3860 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3861 struct ena_admin_ena_hw_hints *hints; 3862 3863 ENA_WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION, adapter->ena_dev, 3864 "Invalid group(%x) expected %x\n", aenq_e->aenq_common_desc.group, 3865 ENA_ADMIN_NOTIFICATION); 3866 3867 switch (aenq_e->aenq_common_desc.syndrome) { 3868 case ENA_ADMIN_UPDATE_HINTS: 3869 hints = 3870 (struct ena_admin_ena_hw_hints *)(&aenq_e->inline_data_w4); 3871 ena_update_hints(adapter, hints); 3872 break; 3873 default: 3874 ena_log(adapter->pdev, ERR, 3875 "Invalid aenq notification link state %d\n", 3876 aenq_e->aenq_common_desc.syndrome); 3877 } 3878 } 3879 3880 static void 3881 ena_lock_init(void *arg) 3882 { 3883 ENA_LOCK_INIT(); 3884 } 3885 SYSINIT(ena_lock_init, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_init, NULL); 3886 3887 static void 3888 ena_lock_uninit(void *arg) 3889 { 3890 ENA_LOCK_DESTROY(); 3891 } 3892 SYSUNINIT(ena_lock_uninit, SI_SUB_LOCK, SI_ORDER_FIRST, ena_lock_uninit, NULL); 3893 3894 /** 3895 * This handler will called for unknown event group or unimplemented handlers 3896 **/ 3897 static void 3898 unimplemented_aenq_handler(void *adapter_data, 3899 struct ena_admin_aenq_entry *aenq_e) 3900 { 3901 struct ena_adapter *adapter = (struct ena_adapter *)adapter_data; 3902 3903 ena_log(adapter->pdev, ERR, 3904 "Unknown event was received or event with unimplemented handler\n"); 3905 } 3906 3907 static struct ena_aenq_handlers aenq_handlers = { 3908 .handlers = { 3909 [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change, 3910 [ENA_ADMIN_NOTIFICATION] = ena_notification, 3911 [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd, 3912 }, 3913 .unimplemented_handler = unimplemented_aenq_handler 3914 }; 3915 3916 /********************************************************************* 3917 * FreeBSD Device Interface Entry Points 3918 *********************************************************************/ 3919 3920 static device_method_t ena_methods[] = { 3921 /* Device interface */ 3922 DEVMETHOD(device_probe, ena_probe), 3923 DEVMETHOD(device_attach, ena_attach), 3924 DEVMETHOD(device_detach, ena_detach), 3925 DEVMETHOD_END 3926 }; 3927 3928 static driver_t ena_driver = { 3929 "ena", ena_methods, sizeof(struct ena_adapter), 3930 }; 3931 3932 devclass_t ena_devclass; 3933 DRIVER_MODULE(ena, pci, ena_driver, ena_devclass, 0, 0); 3934 MODULE_PNP_INFO("U16:vendor;U16:device", pci, ena, ena_vendor_info_array, 3935 nitems(ena_vendor_info_array) - 1); 3936 MODULE_DEPEND(ena, pci, 1, 1, 1); 3937 MODULE_DEPEND(ena, ether, 1, 1, 1); 3938 #ifdef DEV_NETMAP 3939 MODULE_DEPEND(ena, netmap, 1, 1, 1); 3940 #endif /* DEV_NETMAP */ 3941 3942 /*********************************************************************/ 3943