145c98dacSZbigniew Bodek /*- 29eb1615fSMarcin Wojtas * SPDX-License-Identifier: BSD-3-Clause 345c98dacSZbigniew Bodek * 48483b844SMarcin Wojtas * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates. 545c98dacSZbigniew Bodek * All rights reserved. 645c98dacSZbigniew Bodek * 745c98dacSZbigniew Bodek * Redistribution and use in source and binary forms, with or without 845c98dacSZbigniew Bodek * modification, are permitted provided that the following conditions 945c98dacSZbigniew Bodek * are met: 1045c98dacSZbigniew Bodek * 1145c98dacSZbigniew Bodek * * Redistributions of source code must retain the above copyright 1245c98dacSZbigniew Bodek * notice, this list of conditions and the following disclaimer. 1345c98dacSZbigniew Bodek * * Redistributions in binary form must reproduce the above copyright 1445c98dacSZbigniew Bodek * notice, this list of conditions and the following disclaimer in 1545c98dacSZbigniew Bodek * the documentation and/or other materials provided with the 1645c98dacSZbigniew Bodek * distribution. 1745c98dacSZbigniew Bodek * * Neither the name of copyright holder nor the names of its 1845c98dacSZbigniew Bodek * contributors may be used to endorse or promote products derived 1945c98dacSZbigniew Bodek * from this software without specific prior written permission. 2045c98dacSZbigniew Bodek * 2145c98dacSZbigniew Bodek * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2245c98dacSZbigniew Bodek * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2345c98dacSZbigniew Bodek * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2445c98dacSZbigniew Bodek * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2545c98dacSZbigniew Bodek * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2645c98dacSZbigniew Bodek * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2745c98dacSZbigniew Bodek * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2845c98dacSZbigniew Bodek * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2945c98dacSZbigniew Bodek * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 3045c98dacSZbigniew Bodek * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3145c98dacSZbigniew Bodek * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3245c98dacSZbigniew Bodek */ 3345c98dacSZbigniew Bodek 3445c98dacSZbigniew Bodek #ifndef ENA_PLAT_H_ 3545c98dacSZbigniew Bodek #define ENA_PLAT_H_ 3645c98dacSZbigniew Bodek 3745c98dacSZbigniew Bodek #include <sys/cdefs.h> 3845c98dacSZbigniew Bodek __FBSDID("$FreeBSD$"); 3945c98dacSZbigniew Bodek 4045c98dacSZbigniew Bodek #include <sys/param.h> 4145c98dacSZbigniew Bodek #include <sys/systm.h> 4245c98dacSZbigniew Bodek 4345c98dacSZbigniew Bodek #include <sys/bus.h> 4445c98dacSZbigniew Bodek #include <sys/condvar.h> 4545c98dacSZbigniew Bodek #include <sys/endian.h> 4645c98dacSZbigniew Bodek #include <sys/kernel.h> 4745c98dacSZbigniew Bodek #include <sys/kthread.h> 4845c98dacSZbigniew Bodek #include <sys/malloc.h> 4945c98dacSZbigniew Bodek #include <sys/mbuf.h> 5045c98dacSZbigniew Bodek #include <sys/module.h> 5145c98dacSZbigniew Bodek #include <sys/rman.h> 5245c98dacSZbigniew Bodek #include <sys/proc.h> 5345c98dacSZbigniew Bodek #include <sys/smp.h> 5445c98dacSZbigniew Bodek #include <sys/socket.h> 5545c98dacSZbigniew Bodek #include <sys/sockio.h> 5645c98dacSZbigniew Bodek #include <sys/sysctl.h> 5745c98dacSZbigniew Bodek #include <sys/taskqueue.h> 5845c98dacSZbigniew Bodek #include <sys/eventhandler.h> 5945c98dacSZbigniew Bodek #include <sys/types.h> 6045c98dacSZbigniew Bodek #include <sys/timetc.h> 6145c98dacSZbigniew Bodek #include <sys/cdefs.h> 6245c98dacSZbigniew Bodek 6345c98dacSZbigniew Bodek #include <machine/atomic.h> 6445c98dacSZbigniew Bodek #include <machine/bus.h> 6545c98dacSZbigniew Bodek #include <machine/in_cksum.h> 6645c98dacSZbigniew Bodek #include <machine/pcpu.h> 6745c98dacSZbigniew Bodek #include <machine/resource.h> 688483b844SMarcin Wojtas #include <machine/_inttypes.h> 6945c98dacSZbigniew Bodek 7045c98dacSZbigniew Bodek #include <net/bpf.h> 7145c98dacSZbigniew Bodek #include <net/ethernet.h> 7245c98dacSZbigniew Bodek #include <net/if.h> 7345c98dacSZbigniew Bodek #include <net/if_var.h> 7445c98dacSZbigniew Bodek #include <net/if_arp.h> 7545c98dacSZbigniew Bodek #include <net/if_dl.h> 7645c98dacSZbigniew Bodek #include <net/if_media.h> 7745c98dacSZbigniew Bodek 7845c98dacSZbigniew Bodek #include <net/if_types.h> 7945c98dacSZbigniew Bodek #include <net/if_vlan_var.h> 8045c98dacSZbigniew Bodek 8145c98dacSZbigniew Bodek #include <netinet/in_systm.h> 8245c98dacSZbigniew Bodek #include <netinet/in.h> 8345c98dacSZbigniew Bodek #include <netinet/if_ether.h> 8445c98dacSZbigniew Bodek #include <netinet/ip.h> 8545c98dacSZbigniew Bodek #include <netinet/ip6.h> 8645c98dacSZbigniew Bodek #include <netinet/tcp.h> 8745c98dacSZbigniew Bodek #include <netinet/tcp_lro.h> 8845c98dacSZbigniew Bodek #include <netinet/udp.h> 8945c98dacSZbigniew Bodek 9045c98dacSZbigniew Bodek #include <dev/led/led.h> 9145c98dacSZbigniew Bodek #include <dev/pci/pcivar.h> 9245c98dacSZbigniew Bodek #include <dev/pci/pcireg.h> 9345c98dacSZbigniew Bodek 9445c98dacSZbigniew Bodek extern struct ena_bus_space ebs; 9545c98dacSZbigniew Bodek 9645c98dacSZbigniew Bodek /* Levels */ 9745c98dacSZbigniew Bodek #define ENA_ALERT (1 << 0) /* Alerts are providing more error info. */ 9845c98dacSZbigniew Bodek #define ENA_WARNING (1 << 1) /* Driver output is more error sensitive. */ 9945c98dacSZbigniew Bodek #define ENA_INFO (1 << 2) /* Provides additional driver info. */ 10045c98dacSZbigniew Bodek #define ENA_DBG (1 << 3) /* Driver output for debugging. */ 10145c98dacSZbigniew Bodek /* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. */ 10245c98dacSZbigniew Bodek #define ENA_TXPTH (1 << 4) /* Allows TX path tracing. */ 10345c98dacSZbigniew Bodek #define ENA_RXPTH (1 << 5) /* Allows RX path tracing. */ 10445c98dacSZbigniew Bodek #define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */ 10545c98dacSZbigniew Bodek #define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */ 10645c98dacSZbigniew Bodek #define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */ 107d17b7d87SMarcin Wojtas #define ENA_NETMAP (1 << 9) /* Detailed info about netmap. */ 10845c98dacSZbigniew Bodek 1094f8f476eSMarcin Wojtas #define DEFAULT_ALLOC_ALIGNMENT 8 1104f8f476eSMarcin Wojtas 1114e8acd84SMarcin Wojtas extern int ena_log_level; 11245c98dacSZbigniew Bodek 1139eb1615fSMarcin Wojtas #define container_of(ptr, type, member) \ 1149eb1615fSMarcin Wojtas ({ \ 1159eb1615fSMarcin Wojtas const __typeof(((type *)0)->member) *__p = (ptr); \ 1169eb1615fSMarcin Wojtas (type *)((uintptr_t)__p - offsetof(type, member)); \ 1179eb1615fSMarcin Wojtas }) 1189eb1615fSMarcin Wojtas 1199eb1615fSMarcin Wojtas #define ena_trace_raw(ctx, level, fmt, args...) \ 12045c98dacSZbigniew Bodek do { \ 1219eb1615fSMarcin Wojtas ((void)(ctx)); \ 1224e8acd84SMarcin Wojtas if (((level) & ena_log_level) != (level)) \ 12345c98dacSZbigniew Bodek break; \ 12445c98dacSZbigniew Bodek printf(fmt, ##args); \ 12545c98dacSZbigniew Bodek } while (0) 12645c98dacSZbigniew Bodek 1279eb1615fSMarcin Wojtas #define ena_trace(ctx, level, fmt, args...) \ 1289eb1615fSMarcin Wojtas ena_trace_raw(ctx, level, "%s() [TID:%d]: " \ 12967ec48bbSMarcin Wojtas fmt, __func__, curthread->td_tid, ##args) 13045c98dacSZbigniew Bodek 13145c98dacSZbigniew Bodek 1329eb1615fSMarcin Wojtas #define ena_trc_dbg(ctx, format, arg...) \ 1339eb1615fSMarcin Wojtas ena_trace(ctx, ENA_DBG, format, ##arg) 1349eb1615fSMarcin Wojtas #define ena_trc_info(ctx, format, arg...) \ 1359eb1615fSMarcin Wojtas ena_trace(ctx, ENA_INFO, format, ##arg) 1369eb1615fSMarcin Wojtas #define ena_trc_warn(ctx, format, arg...) \ 1379eb1615fSMarcin Wojtas ena_trace(ctx, ENA_WARNING, format, ##arg) 1389eb1615fSMarcin Wojtas #define ena_trc_err(ctx, format, arg...) \ 1399eb1615fSMarcin Wojtas ena_trace(ctx, ENA_ALERT, format, ##arg) 14045c98dacSZbigniew Bodek 1418483b844SMarcin Wojtas #define unlikely(x) __predict_false(!!(x)) 1428483b844SMarcin Wojtas #define likely(x) __predict_true(!!(x)) 14345c98dacSZbigniew Bodek 14445c98dacSZbigniew Bodek #define __iomem 14545c98dacSZbigniew Bodek #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE) 14645c98dacSZbigniew Bodek 14745c98dacSZbigniew Bodek #define MAX_ERRNO 4095 14845c98dacSZbigniew Bodek #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO) 14945c98dacSZbigniew Bodek 1509eb1615fSMarcin Wojtas #define ENA_WARN(cond, ctx, format, arg...) \ 15145c98dacSZbigniew Bodek do { \ 15245c98dacSZbigniew Bodek if (unlikely((cond))) { \ 1539eb1615fSMarcin Wojtas ena_trc_warn(ctx, format, ##arg); \ 15445c98dacSZbigniew Bodek } \ 15545c98dacSZbigniew Bodek } while (0) 15645c98dacSZbigniew Bodek 15745c98dacSZbigniew Bodek static inline long IS_ERR(const void *ptr) 15845c98dacSZbigniew Bodek { 15945c98dacSZbigniew Bodek return IS_ERR_VALUE((unsigned long)ptr); 16045c98dacSZbigniew Bodek } 16145c98dacSZbigniew Bodek 16245c98dacSZbigniew Bodek static inline void *ERR_PTR(long error) 16345c98dacSZbigniew Bodek { 16445c98dacSZbigniew Bodek return (void *)error; 16545c98dacSZbigniew Bodek } 16645c98dacSZbigniew Bodek 16745c98dacSZbigniew Bodek static inline long PTR_ERR(const void *ptr) 16845c98dacSZbigniew Bodek { 16945c98dacSZbigniew Bodek return (long) ptr; 17045c98dacSZbigniew Bodek } 17145c98dacSZbigniew Bodek 17267ec48bbSMarcin Wojtas #define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h)))) 17345c98dacSZbigniew Bodek #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h)))) 1746461d6a3SMarcin Wojtas #define BIT(x) (1UL << (x)) 17545c98dacSZbigniew Bodek 17645c98dacSZbigniew Bodek #define ENA_ABORT() BUG() 17745c98dacSZbigniew Bodek #define BUG() panic("ENA BUG") 17845c98dacSZbigniew Bodek 17945c98dacSZbigniew Bodek #define SZ_256 (256) 18045c98dacSZbigniew Bodek #define SZ_4K (4096) 18145c98dacSZbigniew Bodek 18245c98dacSZbigniew Bodek #define ENA_COM_OK 0 18345c98dacSZbigniew Bodek #define ENA_COM_FAULT EFAULT 18445c98dacSZbigniew Bodek #define ENA_COM_INVAL EINVAL 18545c98dacSZbigniew Bodek #define ENA_COM_NO_MEM ENOMEM 18645c98dacSZbigniew Bodek #define ENA_COM_NO_SPACE ENOSPC 18745c98dacSZbigniew Bodek #define ENA_COM_TRY_AGAIN -1 188a195fab0SMarcin Wojtas #define ENA_COM_UNSUPPORTED EOPNOTSUPP 18945c98dacSZbigniew Bodek #define ENA_COM_NO_DEVICE ENODEV 19045c98dacSZbigniew Bodek #define ENA_COM_PERMISSION EPERM 19145c98dacSZbigniew Bodek #define ENA_COM_TIMER_EXPIRED ETIMEDOUT 1929eb1615fSMarcin Wojtas #define ENA_COM_EIO EIO 19345c98dacSZbigniew Bodek 19445c98dacSZbigniew Bodek #define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0) 1958483b844SMarcin Wojtas #define ENA_USLEEP(x) pause_sbt("ena", SBT_1US * (x), SBT_1US, 0) 19645c98dacSZbigniew Bodek #define ENA_UDELAY(x) DELAY(x) 19745c98dacSZbigniew Bodek #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \ 19845c98dacSZbigniew Bodek ((long)cputick2usec(cpu_ticks()) + (timeout_us)) 1998483b844SMarcin Wojtas #define ENA_TIME_EXPIRE(timeout) ((timeout) < cputick2usec(cpu_ticks())) 20045c98dacSZbigniew Bodek #define ENA_MIGHT_SLEEP() 20145c98dacSZbigniew Bodek 20245c98dacSZbigniew Bodek #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y)) 20345c98dacSZbigniew Bodek #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y)) 20445c98dacSZbigniew Bodek 20545c98dacSZbigniew Bodek #define ENA_MIN32(x,y) MIN(x, y) 20645c98dacSZbigniew Bodek #define ENA_MIN16(x,y) MIN(x, y) 20745c98dacSZbigniew Bodek #define ENA_MIN8(x,y) MIN(x, y) 20845c98dacSZbigniew Bodek 20945c98dacSZbigniew Bodek #define ENA_MAX32(x,y) MAX(x, y) 21045c98dacSZbigniew Bodek #define ENA_MAX16(x,y) MAX(x, y) 21145c98dacSZbigniew Bodek #define ENA_MAX8(x,y) MAX(x, y) 21245c98dacSZbigniew Bodek 21345c98dacSZbigniew Bodek /* Spinlock related methods */ 21445c98dacSZbigniew Bodek #define ena_spinlock_t struct mtx 21545c98dacSZbigniew Bodek #define ENA_SPINLOCK_INIT(spinlock) \ 21645c98dacSZbigniew Bodek mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN) 21745c98dacSZbigniew Bodek #define ENA_SPINLOCK_DESTROY(spinlock) \ 21845c98dacSZbigniew Bodek do { \ 21945c98dacSZbigniew Bodek if (mtx_initialized(&(spinlock))) \ 22045c98dacSZbigniew Bodek mtx_destroy(&(spinlock)); \ 22145c98dacSZbigniew Bodek } while (0) 22245c98dacSZbigniew Bodek #define ENA_SPINLOCK_LOCK(spinlock, flags) \ 22345c98dacSZbigniew Bodek do { \ 22445c98dacSZbigniew Bodek (void)(flags); \ 22545c98dacSZbigniew Bodek mtx_lock_spin(&(spinlock)); \ 22645c98dacSZbigniew Bodek } while (0) 22745c98dacSZbigniew Bodek #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \ 22845c98dacSZbigniew Bodek do { \ 22945c98dacSZbigniew Bodek (void)(flags); \ 23045c98dacSZbigniew Bodek mtx_unlock_spin(&(spinlock)); \ 23145c98dacSZbigniew Bodek } while (0) 23245c98dacSZbigniew Bodek 23345c98dacSZbigniew Bodek 23445c98dacSZbigniew Bodek /* Wait queue related methods */ 23545c98dacSZbigniew Bodek #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; } 23645c98dacSZbigniew Bodek #define ENA_WAIT_EVENT_INIT(waitqueue) \ 23745c98dacSZbigniew Bodek do { \ 23845c98dacSZbigniew Bodek cv_init(&((waitqueue).wq), "cv"); \ 23945c98dacSZbigniew Bodek mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \ 24045c98dacSZbigniew Bodek } while (0) 2419eb1615fSMarcin Wojtas #define ENA_WAIT_EVENTS_DESTROY(admin_queue) \ 24245c98dacSZbigniew Bodek do { \ 2439eb1615fSMarcin Wojtas struct ena_comp_ctx *comp_ctx; \ 2449eb1615fSMarcin Wojtas int i; \ 2459eb1615fSMarcin Wojtas for (i = 0; i < admin_queue->q_depth; i++) { \ 2469eb1615fSMarcin Wojtas comp_ctx = get_comp_ctxt(admin_queue, i, false); \ 2479eb1615fSMarcin Wojtas if (comp_ctx != NULL) { \ 2489eb1615fSMarcin Wojtas cv_destroy(&((comp_ctx->wait_event).wq)); \ 2499eb1615fSMarcin Wojtas mtx_destroy(&((comp_ctx->wait_event).mtx)); \ 2509eb1615fSMarcin Wojtas } \ 2519eb1615fSMarcin Wojtas } \ 25245c98dacSZbigniew Bodek } while (0) 25345c98dacSZbigniew Bodek #define ENA_WAIT_EVENT_CLEAR(waitqueue) \ 25445c98dacSZbigniew Bodek cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description) 25545c98dacSZbigniew Bodek #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \ 25645c98dacSZbigniew Bodek do { \ 25745c98dacSZbigniew Bodek mtx_lock(&((waitqueue).mtx)); \ 25845c98dacSZbigniew Bodek cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \ 25945c98dacSZbigniew Bodek timeout_us * hz / 1000 / 1000 ); \ 26045c98dacSZbigniew Bodek mtx_unlock(&((waitqueue).mtx)); \ 26145c98dacSZbigniew Bodek } while (0) 2626461d6a3SMarcin Wojtas #define ENA_WAIT_EVENT_SIGNAL(waitqueue) \ 2636461d6a3SMarcin Wojtas do { \ 2646461d6a3SMarcin Wojtas mtx_lock(&((waitqueue).mtx)); \ 2656461d6a3SMarcin Wojtas cv_broadcast(&((waitqueue).wq)); \ 2666461d6a3SMarcin Wojtas mtx_unlock(&((waitqueue).mtx)); \ 2676461d6a3SMarcin Wojtas } while (0) 26845c98dacSZbigniew Bodek 26945c98dacSZbigniew Bodek #define dma_addr_t bus_addr_t 27045c98dacSZbigniew Bodek #define u8 uint8_t 27145c98dacSZbigniew Bodek #define u16 uint16_t 27245c98dacSZbigniew Bodek #define u32 uint32_t 27345c98dacSZbigniew Bodek #define u64 uint64_t 27445c98dacSZbigniew Bodek 27545c98dacSZbigniew Bodek typedef struct { 27645c98dacSZbigniew Bodek bus_addr_t paddr; 27745c98dacSZbigniew Bodek caddr_t vaddr; 27845c98dacSZbigniew Bodek bus_dma_tag_t tag; 27945c98dacSZbigniew Bodek bus_dmamap_t map; 28045c98dacSZbigniew Bodek bus_dma_segment_t seg; 28145c98dacSZbigniew Bodek int nseg; 28245c98dacSZbigniew Bodek } ena_mem_handle_t; 28345c98dacSZbigniew Bodek 28445c98dacSZbigniew Bodek struct ena_bus { 28545c98dacSZbigniew Bodek bus_space_handle_t reg_bar_h; 28645c98dacSZbigniew Bodek bus_space_tag_t reg_bar_t; 28745c98dacSZbigniew Bodek bus_space_handle_t mem_bar_h; 28845c98dacSZbigniew Bodek bus_space_tag_t mem_bar_t; 28945c98dacSZbigniew Bodek }; 29045c98dacSZbigniew Bodek 29145c98dacSZbigniew Bodek typedef uint32_t ena_atomic32_t; 29245c98dacSZbigniew Bodek 2938483b844SMarcin Wojtas #define ENA_PRIu64 PRIu64 2948483b844SMarcin Wojtas 2958483b844SMarcin Wojtas typedef uint64_t ena_time_t; 2969eb1615fSMarcin Wojtas typedef struct ifnet ena_netdev; 2978483b844SMarcin Wojtas 29845c98dacSZbigniew Bodek void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, 29945c98dacSZbigniew Bodek int error); 30045c98dacSZbigniew Bodek int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma, 3014f8f476eSMarcin Wojtas int mapflags, bus_size_t alignment); 30245c98dacSZbigniew Bodek 3038483b844SMarcin Wojtas static inline uint32_t 3048483b844SMarcin Wojtas ena_reg_read32(struct ena_bus *bus, bus_size_t offset) 3058483b844SMarcin Wojtas { 3068483b844SMarcin Wojtas uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset); 3078483b844SMarcin Wojtas rmb(); 3088483b844SMarcin Wojtas return v; 3098483b844SMarcin Wojtas } 3108483b844SMarcin Wojtas 311a195fab0SMarcin Wojtas #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \ 312a195fab0SMarcin Wojtas do { \ 313a195fab0SMarcin Wojtas int count, i; \ 314a195fab0SMarcin Wojtas volatile uint64_t *to = (volatile uint64_t *)(dst); \ 315a195fab0SMarcin Wojtas const uint64_t *from = (const uint64_t *)(src); \ 316a195fab0SMarcin Wojtas count = (size) / 8; \ 317a195fab0SMarcin Wojtas \ 318a195fab0SMarcin Wojtas for (i = 0; i < count; i++, from++, to++) \ 319a195fab0SMarcin Wojtas *to = *from; \ 320a195fab0SMarcin Wojtas } while (0) 321a195fab0SMarcin Wojtas 32245c98dacSZbigniew Bodek #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO) 32345c98dacSZbigniew Bodek #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL) 3248483b844SMarcin Wojtas #define ENA_MEM_FREE(dmadev, ptr, size) \ 3258483b844SMarcin Wojtas do { \ 3268483b844SMarcin Wojtas (void)(size); \ 3278483b844SMarcin Wojtas free(ptr, M_DEVBUF); \ 3288483b844SMarcin Wojtas } while (0) 3294f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \ 3304f8f476eSMarcin Wojtas handle, node, dev_node, alignment) \ 33145c98dacSZbigniew Bodek do { \ 33245c98dacSZbigniew Bodek ((virt) = NULL); \ 33345c98dacSZbigniew Bodek (void)(dev_node); \ 33445c98dacSZbigniew Bodek } while (0) 33545c98dacSZbigniew Bodek 3364f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, \ 3374f8f476eSMarcin Wojtas node, dev_node) \ 3384f8f476eSMarcin Wojtas ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, \ 3394f8f476eSMarcin Wojtas phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT) 3404f8f476eSMarcin Wojtas 3414f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma, \ 3424f8f476eSMarcin Wojtas alignment) \ 34345c98dacSZbigniew Bodek do { \ 3444f8f476eSMarcin Wojtas ena_dma_alloc((dmadev), (size), &(dma), 0, alignment); \ 34545c98dacSZbigniew Bodek (virt) = (void *)(dma).vaddr; \ 34645c98dacSZbigniew Bodek (phys) = (dma).paddr; \ 34745c98dacSZbigniew Bodek } while (0) 34845c98dacSZbigniew Bodek 3494f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \ 3504f8f476eSMarcin Wojtas ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, \ 3514f8f476eSMarcin Wojtas phys, dma, DEFAULT_ALLOC_ALIGNMENT) 3524f8f476eSMarcin Wojtas 35345c98dacSZbigniew Bodek #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \ 35445c98dacSZbigniew Bodek do { \ 35545c98dacSZbigniew Bodek (void)size; \ 35645c98dacSZbigniew Bodek bus_dmamap_unload((dma).tag, (dma).map); \ 35745c98dacSZbigniew Bodek bus_dmamem_free((dma).tag, (virt), (dma).map); \ 35845c98dacSZbigniew Bodek bus_dma_tag_destroy((dma).tag); \ 35945c98dacSZbigniew Bodek (dma).tag = NULL; \ 36045c98dacSZbigniew Bodek (virt) = NULL; \ 36145c98dacSZbigniew Bodek } while (0) 36245c98dacSZbigniew Bodek 36345c98dacSZbigniew Bodek /* Register R/W methods */ 36445c98dacSZbigniew Bodek #define ENA_REG_WRITE32(bus, value, offset) \ 3658483b844SMarcin Wojtas do { \ 3668483b844SMarcin Wojtas wmb(); \ 3678483b844SMarcin Wojtas ENA_REG_WRITE32_RELAXED(bus, value, offset); \ 3688483b844SMarcin Wojtas } while (0) 3698483b844SMarcin Wojtas 3708483b844SMarcin Wojtas #define ENA_REG_WRITE32_RELAXED(bus, value, offset) \ 37145c98dacSZbigniew Bodek bus_space_write_4( \ 37245c98dacSZbigniew Bodek ((struct ena_bus*)bus)->reg_bar_t, \ 37345c98dacSZbigniew Bodek ((struct ena_bus*)bus)->reg_bar_h, \ 37445c98dacSZbigniew Bodek (bus_size_t)(offset), (value)) 37545c98dacSZbigniew Bodek 37645c98dacSZbigniew Bodek #define ENA_REG_READ32(bus, offset) \ 3778483b844SMarcin Wojtas ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset)) 37845c98dacSZbigniew Bodek 37967ec48bbSMarcin Wojtas #define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync( \ 38067ec48bbSMarcin Wojtas (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE) 38167ec48bbSMarcin Wojtas #define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync( \ 38267ec48bbSMarcin Wojtas (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD) 38367ec48bbSMarcin Wojtas #define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync( \ 38467ec48bbSMarcin Wojtas (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD) 38567ec48bbSMarcin Wojtas #define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle) 386a195fab0SMarcin Wojtas 38745c98dacSZbigniew Bodek #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0) 38845c98dacSZbigniew Bodek 38945c98dacSZbigniew Bodek #define VLAN_HLEN sizeof(struct ether_vlan_header) 39045c98dacSZbigniew Bodek #define CSUM_OFFLOAD (CSUM_IP|CSUM_TCP|CSUM_UDP) 39145c98dacSZbigniew Bodek 39267ec48bbSMarcin Wojtas #define prefetch(x) (void)(x) 39367ec48bbSMarcin Wojtas #define prefetchw(x) (void)(x) 39445c98dacSZbigniew Bodek 39545c98dacSZbigniew Bodek /* DMA buffers access */ 39645c98dacSZbigniew Bodek #define dma_unmap_addr(p, name) ((p)->dma->name) 39745c98dacSZbigniew Bodek #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v)) 39845c98dacSZbigniew Bodek #define dma_unmap_len(p, name) ((p)->name) 39945c98dacSZbigniew Bodek #define dma_unmap_len_set(p, name, v) (((p)->name) = (v)) 40045c98dacSZbigniew Bodek 40145c98dacSZbigniew Bodek #define memcpy_toio memcpy 40245c98dacSZbigniew Bodek 40345c98dacSZbigniew Bodek #define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1) 40445c98dacSZbigniew Bodek #define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1) 40545c98dacSZbigniew Bodek #define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR) 40645c98dacSZbigniew Bodek #define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL) 40745c98dacSZbigniew Bodek 40845c98dacSZbigniew Bodek #define barrier() __asm__ __volatile__("": : :"memory") 40967ec48bbSMarcin Wojtas #define dma_rmb() barrier() 41067ec48bbSMarcin Wojtas #define mmiowb() barrier() 41167ec48bbSMarcin Wojtas 41245c98dacSZbigniew Bodek #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x)) 41345c98dacSZbigniew Bodek #define READ_ONCE(x) ({ \ 41445c98dacSZbigniew Bodek __typeof(x) __var; \ 41545c98dacSZbigniew Bodek barrier(); \ 41645c98dacSZbigniew Bodek __var = ACCESS_ONCE(x); \ 41745c98dacSZbigniew Bodek barrier(); \ 41845c98dacSZbigniew Bodek __var; \ 41945c98dacSZbigniew Bodek }) 42067ec48bbSMarcin Wojtas #define READ_ONCE8(x) READ_ONCE(x) 42167ec48bbSMarcin Wojtas #define READ_ONCE16(x) READ_ONCE(x) 42267ec48bbSMarcin Wojtas #define READ_ONCE32(x) READ_ONCE(x) 42367ec48bbSMarcin Wojtas 42467ec48bbSMarcin Wojtas #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16)) 42567ec48bbSMarcin Wojtas #define lower_32_bits(n) ((uint32_t)(n)) 42667ec48bbSMarcin Wojtas 42767ec48bbSMarcin Wojtas #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) 42845c98dacSZbigniew Bodek 4298483b844SMarcin Wojtas #define ENA_FFS(x) ffs(x) 4308483b844SMarcin Wojtas 4318483b844SMarcin Wojtas void ena_rss_key_fill(void *key, size_t size); 4328483b844SMarcin Wojtas 4338483b844SMarcin Wojtas #define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size) 4348483b844SMarcin Wojtas 435a195fab0SMarcin Wojtas #include "ena_defs/ena_includes.h" 43645c98dacSZbigniew Bodek 43745c98dacSZbigniew Bodek #endif /* ENA_PLAT_H_ */ 438