145c98dacSZbigniew Bodek /*-
29eb1615fSMarcin Wojtas * SPDX-License-Identifier: BSD-3-Clause
345c98dacSZbigniew Bodek *
4adfed2d8SArthur Kiyanovski * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates.
545c98dacSZbigniew Bodek * All rights reserved.
645c98dacSZbigniew Bodek *
745c98dacSZbigniew Bodek * Redistribution and use in source and binary forms, with or without
845c98dacSZbigniew Bodek * modification, are permitted provided that the following conditions
945c98dacSZbigniew Bodek * are met:
1045c98dacSZbigniew Bodek *
1145c98dacSZbigniew Bodek * * Redistributions of source code must retain the above copyright
1245c98dacSZbigniew Bodek * notice, this list of conditions and the following disclaimer.
1345c98dacSZbigniew Bodek * * Redistributions in binary form must reproduce the above copyright
1445c98dacSZbigniew Bodek * notice, this list of conditions and the following disclaimer in
1545c98dacSZbigniew Bodek * the documentation and/or other materials provided with the
1645c98dacSZbigniew Bodek * distribution.
1745c98dacSZbigniew Bodek * * Neither the name of copyright holder nor the names of its
1845c98dacSZbigniew Bodek * contributors may be used to endorse or promote products derived
1945c98dacSZbigniew Bodek * from this software without specific prior written permission.
2045c98dacSZbigniew Bodek *
2145c98dacSZbigniew Bodek * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
2245c98dacSZbigniew Bodek * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
2345c98dacSZbigniew Bodek * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
2445c98dacSZbigniew Bodek * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
2545c98dacSZbigniew Bodek * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
2645c98dacSZbigniew Bodek * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
2745c98dacSZbigniew Bodek * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2845c98dacSZbigniew Bodek * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2945c98dacSZbigniew Bodek * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
3045c98dacSZbigniew Bodek * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
3145c98dacSZbigniew Bodek * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3245c98dacSZbigniew Bodek */
3345c98dacSZbigniew Bodek
3445c98dacSZbigniew Bodek #ifndef ENA_PLAT_H_
3545c98dacSZbigniew Bodek #define ENA_PLAT_H_
3645c98dacSZbigniew Bodek
3745c98dacSZbigniew Bodek #include <sys/cdefs.h>
3845c98dacSZbigniew Bodek __FBSDID("$FreeBSD$");
3945c98dacSZbigniew Bodek
4045c98dacSZbigniew Bodek #include <sys/param.h>
4145c98dacSZbigniew Bodek #include <sys/systm.h>
4245c98dacSZbigniew Bodek
4345c98dacSZbigniew Bodek #include <sys/bus.h>
4445c98dacSZbigniew Bodek #include <sys/condvar.h>
45eb4c4f4aSMarcin Wojtas #include <sys/domainset.h>
4645c98dacSZbigniew Bodek #include <sys/endian.h>
4745c98dacSZbigniew Bodek #include <sys/kernel.h>
4845c98dacSZbigniew Bodek #include <sys/kthread.h>
4945c98dacSZbigniew Bodek #include <sys/malloc.h>
5045c98dacSZbigniew Bodek #include <sys/mbuf.h>
5145c98dacSZbigniew Bodek #include <sys/module.h>
5245c98dacSZbigniew Bodek #include <sys/rman.h>
5345c98dacSZbigniew Bodek #include <sys/proc.h>
5445c98dacSZbigniew Bodek #include <sys/smp.h>
5545c98dacSZbigniew Bodek #include <sys/socket.h>
5645c98dacSZbigniew Bodek #include <sys/sockio.h>
5745c98dacSZbigniew Bodek #include <sys/sysctl.h>
5845c98dacSZbigniew Bodek #include <sys/taskqueue.h>
5945c98dacSZbigniew Bodek #include <sys/eventhandler.h>
6045c98dacSZbigniew Bodek #include <sys/types.h>
6145c98dacSZbigniew Bodek #include <sys/timetc.h>
6245c98dacSZbigniew Bodek #include <sys/cdefs.h>
6345c98dacSZbigniew Bodek
6445c98dacSZbigniew Bodek #include <machine/atomic.h>
6545c98dacSZbigniew Bodek #include <machine/bus.h>
6645c98dacSZbigniew Bodek #include <machine/in_cksum.h>
6745c98dacSZbigniew Bodek #include <machine/pcpu.h>
6845c98dacSZbigniew Bodek #include <machine/resource.h>
698483b844SMarcin Wojtas #include <machine/_inttypes.h>
7045c98dacSZbigniew Bodek
7145c98dacSZbigniew Bodek #include <net/bpf.h>
7245c98dacSZbigniew Bodek #include <net/ethernet.h>
7345c98dacSZbigniew Bodek #include <net/if.h>
7445c98dacSZbigniew Bodek #include <net/if_var.h>
7545c98dacSZbigniew Bodek #include <net/if_arp.h>
7645c98dacSZbigniew Bodek #include <net/if_dl.h>
7745c98dacSZbigniew Bodek #include <net/if_media.h>
7845c98dacSZbigniew Bodek
7945c98dacSZbigniew Bodek #include <net/if_types.h>
8045c98dacSZbigniew Bodek #include <net/if_vlan_var.h>
8145c98dacSZbigniew Bodek
8245c98dacSZbigniew Bodek #include <netinet/in_systm.h>
8345c98dacSZbigniew Bodek #include <netinet/in.h>
8445c98dacSZbigniew Bodek #include <netinet/if_ether.h>
8545c98dacSZbigniew Bodek #include <netinet/ip.h>
8645c98dacSZbigniew Bodek #include <netinet/ip6.h>
8745c98dacSZbigniew Bodek #include <netinet/tcp.h>
8845c98dacSZbigniew Bodek #include <netinet/tcp_lro.h>
8945c98dacSZbigniew Bodek #include <netinet/udp.h>
9045c98dacSZbigniew Bodek
9145c98dacSZbigniew Bodek #include <dev/led/led.h>
9245c98dacSZbigniew Bodek #include <dev/pci/pcivar.h>
9345c98dacSZbigniew Bodek #include <dev/pci/pcireg.h>
9445c98dacSZbigniew Bodek
95adfed2d8SArthur Kiyanovski enum ena_log_t {
96adfed2d8SArthur Kiyanovski ENA_ERR = 0,
97adfed2d8SArthur Kiyanovski ENA_WARN,
98adfed2d8SArthur Kiyanovski ENA_INFO,
99adfed2d8SArthur Kiyanovski ENA_DBG,
100adfed2d8SArthur Kiyanovski };
101adfed2d8SArthur Kiyanovski
102adfed2d8SArthur Kiyanovski extern int ena_log_level;
103adfed2d8SArthur Kiyanovski
104adfed2d8SArthur Kiyanovski #define ena_log(dev, level, fmt, args...) \
105adfed2d8SArthur Kiyanovski do { \
106adfed2d8SArthur Kiyanovski if (ENA_ ## level <= ena_log_level) \
107adfed2d8SArthur Kiyanovski device_printf((dev), fmt, ##args); \
108adfed2d8SArthur Kiyanovski } while (0)
109adfed2d8SArthur Kiyanovski
110adfed2d8SArthur Kiyanovski #define ena_log_raw(level, fmt, args...) \
111adfed2d8SArthur Kiyanovski do { \
112adfed2d8SArthur Kiyanovski if (ENA_ ## level <= ena_log_level) \
113adfed2d8SArthur Kiyanovski printf(fmt, ##args); \
114adfed2d8SArthur Kiyanovski } while (0)
115adfed2d8SArthur Kiyanovski
116adfed2d8SArthur Kiyanovski #define ena_log_unused(dev, level, fmt, args...) \
117adfed2d8SArthur Kiyanovski do { \
118adfed2d8SArthur Kiyanovski (void)(dev); \
119adfed2d8SArthur Kiyanovski } while (0)
120adfed2d8SArthur Kiyanovski
121adfed2d8SArthur Kiyanovski #ifdef ENA_LOG_IO_ENABLE
122adfed2d8SArthur Kiyanovski #define ena_log_io(dev, level, fmt, args...) \
123adfed2d8SArthur Kiyanovski ena_log((dev), level, fmt, ##args)
124adfed2d8SArthur Kiyanovski #else
125adfed2d8SArthur Kiyanovski #define ena_log_io(dev, level, fmt, args...) \
126adfed2d8SArthur Kiyanovski ena_log_unused((dev), level, fmt, ##args)
127adfed2d8SArthur Kiyanovski #endif
128adfed2d8SArthur Kiyanovski
129adfed2d8SArthur Kiyanovski #define ena_log_nm(dev, level, fmt, args...) \
130adfed2d8SArthur Kiyanovski ena_log((dev), level, "[nm] " fmt, ##args)
1313fc5d816SMarcin Wojtas
13245c98dacSZbigniew Bodek extern struct ena_bus_space ebs;
13345c98dacSZbigniew Bodek
1344f8f476eSMarcin Wojtas #define DEFAULT_ALLOC_ALIGNMENT 8
1353fc5d816SMarcin Wojtas #define ENA_CDESC_RING_SIZE_ALIGNMENT (1 << 12) /* 4K */
1364f8f476eSMarcin Wojtas
1379eb1615fSMarcin Wojtas #define container_of(ptr, type, member) \
1389eb1615fSMarcin Wojtas ({ \
1399eb1615fSMarcin Wojtas const __typeof(((type *)0)->member) *__p = (ptr); \
1409eb1615fSMarcin Wojtas (type *)((uintptr_t)__p - offsetof(type, member)); \
1419eb1615fSMarcin Wojtas })
1429eb1615fSMarcin Wojtas
1439eb1615fSMarcin Wojtas #define ena_trace(ctx, level, fmt, args...) \
1443fc5d816SMarcin Wojtas ena_log((ctx)->dmadev, level, "%s() [TID:%d]: " \
14567ec48bbSMarcin Wojtas fmt, __func__, curthread->td_tid, ##args)
14645c98dacSZbigniew Bodek
1479eb1615fSMarcin Wojtas #define ena_trc_dbg(ctx, format, arg...) \
1483fc5d816SMarcin Wojtas ena_trace(ctx, DBG, format, ##arg)
1499eb1615fSMarcin Wojtas #define ena_trc_info(ctx, format, arg...) \
1503fc5d816SMarcin Wojtas ena_trace(ctx, INFO, format, ##arg)
1519eb1615fSMarcin Wojtas #define ena_trc_warn(ctx, format, arg...) \
1523fc5d816SMarcin Wojtas ena_trace(ctx, WARN, format, ##arg)
1539eb1615fSMarcin Wojtas #define ena_trc_err(ctx, format, arg...) \
1543fc5d816SMarcin Wojtas ena_trace(ctx, ERR, format, ##arg)
15545c98dacSZbigniew Bodek
1568483b844SMarcin Wojtas #define unlikely(x) __predict_false(!!(x))
1578483b844SMarcin Wojtas #define likely(x) __predict_true(!!(x))
15845c98dacSZbigniew Bodek
15945c98dacSZbigniew Bodek #define __iomem
16045c98dacSZbigniew Bodek #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE)
16145c98dacSZbigniew Bodek
16245c98dacSZbigniew Bodek #define MAX_ERRNO 4095
16345c98dacSZbigniew Bodek #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
16445c98dacSZbigniew Bodek
1659eb1615fSMarcin Wojtas #define ENA_WARN(cond, ctx, format, arg...) \
16645c98dacSZbigniew Bodek do { \
16745c98dacSZbigniew Bodek if (unlikely((cond))) { \
1689eb1615fSMarcin Wojtas ena_trc_warn(ctx, format, ##arg); \
16945c98dacSZbigniew Bodek } \
17045c98dacSZbigniew Bodek } while (0)
17145c98dacSZbigniew Bodek
IS_ERR(const void * ptr)17245c98dacSZbigniew Bodek static inline long IS_ERR(const void *ptr)
17345c98dacSZbigniew Bodek {
17445c98dacSZbigniew Bodek return IS_ERR_VALUE((unsigned long)ptr);
17545c98dacSZbigniew Bodek }
17645c98dacSZbigniew Bodek
ERR_PTR(long error)17745c98dacSZbigniew Bodek static inline void *ERR_PTR(long error)
17845c98dacSZbigniew Bodek {
17945c98dacSZbigniew Bodek return (void *)error;
18045c98dacSZbigniew Bodek }
18145c98dacSZbigniew Bodek
PTR_ERR(const void * ptr)18245c98dacSZbigniew Bodek static inline long PTR_ERR(const void *ptr)
18345c98dacSZbigniew Bodek {
18445c98dacSZbigniew Bodek return (long) ptr;
18545c98dacSZbigniew Bodek }
18645c98dacSZbigniew Bodek
18767ec48bbSMarcin Wojtas #define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h))))
18845c98dacSZbigniew Bodek #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
1896461d6a3SMarcin Wojtas #define BIT(x) (1UL << (x))
190adfed2d8SArthur Kiyanovski #define BIT64(x) BIT(x)
19145c98dacSZbigniew Bodek #define ENA_ABORT() BUG()
19245c98dacSZbigniew Bodek #define BUG() panic("ENA BUG")
19345c98dacSZbigniew Bodek
19445c98dacSZbigniew Bodek #define SZ_256 (256)
19545c98dacSZbigniew Bodek #define SZ_4K (4096)
19645c98dacSZbigniew Bodek
19745c98dacSZbigniew Bodek #define ENA_COM_OK 0
19845c98dacSZbigniew Bodek #define ENA_COM_FAULT EFAULT
19945c98dacSZbigniew Bodek #define ENA_COM_INVAL EINVAL
20045c98dacSZbigniew Bodek #define ENA_COM_NO_MEM ENOMEM
20145c98dacSZbigniew Bodek #define ENA_COM_NO_SPACE ENOSPC
20245c98dacSZbigniew Bodek #define ENA_COM_TRY_AGAIN -1
203a195fab0SMarcin Wojtas #define ENA_COM_UNSUPPORTED EOPNOTSUPP
20445c98dacSZbigniew Bodek #define ENA_COM_NO_DEVICE ENODEV
20545c98dacSZbigniew Bodek #define ENA_COM_PERMISSION EPERM
20645c98dacSZbigniew Bodek #define ENA_COM_TIMER_EXPIRED ETIMEDOUT
2079eb1615fSMarcin Wojtas #define ENA_COM_EIO EIO
208adfed2d8SArthur Kiyanovski #define ENA_COM_DEVICE_BUSY EBUSY
20945c98dacSZbigniew Bodek
210eb4c4f4aSMarcin Wojtas #define ENA_NODE_ANY (-1)
211eb4c4f4aSMarcin Wojtas
21245c98dacSZbigniew Bodek #define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
2138483b844SMarcin Wojtas #define ENA_USLEEP(x) pause_sbt("ena", SBT_1US * (x), SBT_1US, 0)
21445c98dacSZbigniew Bodek #define ENA_UDELAY(x) DELAY(x)
21545c98dacSZbigniew Bodek #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \
21645c98dacSZbigniew Bodek ((long)cputick2usec(cpu_ticks()) + (timeout_us))
2178483b844SMarcin Wojtas #define ENA_TIME_EXPIRE(timeout) ((timeout) < cputick2usec(cpu_ticks()))
218adfed2d8SArthur Kiyanovski #define ENA_TIME_EXPIRE_HIGH_RES ENA_TIME_EXPIRE
219adfed2d8SArthur Kiyanovski #define ENA_TIME_INIT_HIGH_RES() (0)
220adfed2d8SArthur Kiyanovski #define ENA_TIME_COMPARE_HIGH_RES(time1, time2) \
221adfed2d8SArthur Kiyanovski ((time1 < time2) ? -1 : ((time1 > time2) ? 1 : 0))
222adfed2d8SArthur Kiyanovski #define ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(current_time, timeout_us) \
223adfed2d8SArthur Kiyanovski ((long)cputick2usec(cpu_ticks()) + (timeout_us))
224adfed2d8SArthur Kiyanovski #define ENA_GET_SYSTEM_TIME_HIGH_RES() ENA_GET_SYSTEM_TIMEOUT(0)
22545c98dacSZbigniew Bodek #define ENA_MIGHT_SLEEP()
22645c98dacSZbigniew Bodek
22745c98dacSZbigniew Bodek #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
22845c98dacSZbigniew Bodek #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y))
22945c98dacSZbigniew Bodek
23045c98dacSZbigniew Bodek #define ENA_MIN32(x,y) MIN(x, y)
23145c98dacSZbigniew Bodek #define ENA_MIN16(x,y) MIN(x, y)
23245c98dacSZbigniew Bodek #define ENA_MIN8(x,y) MIN(x, y)
23345c98dacSZbigniew Bodek
23445c98dacSZbigniew Bodek #define ENA_MAX32(x,y) MAX(x, y)
23545c98dacSZbigniew Bodek #define ENA_MAX16(x,y) MAX(x, y)
23645c98dacSZbigniew Bodek #define ENA_MAX8(x,y) MAX(x, y)
23745c98dacSZbigniew Bodek
23845c98dacSZbigniew Bodek /* Spinlock related methods */
23945c98dacSZbigniew Bodek #define ena_spinlock_t struct mtx
24045c98dacSZbigniew Bodek #define ENA_SPINLOCK_INIT(spinlock) \
24145c98dacSZbigniew Bodek mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN)
24245c98dacSZbigniew Bodek #define ENA_SPINLOCK_DESTROY(spinlock) \
24345c98dacSZbigniew Bodek do { \
24445c98dacSZbigniew Bodek if (mtx_initialized(&(spinlock))) \
24545c98dacSZbigniew Bodek mtx_destroy(&(spinlock)); \
24645c98dacSZbigniew Bodek } while (0)
24745c98dacSZbigniew Bodek #define ENA_SPINLOCK_LOCK(spinlock, flags) \
24845c98dacSZbigniew Bodek do { \
24945c98dacSZbigniew Bodek (void)(flags); \
25045c98dacSZbigniew Bodek mtx_lock_spin(&(spinlock)); \
25145c98dacSZbigniew Bodek } while (0)
25245c98dacSZbigniew Bodek #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \
25345c98dacSZbigniew Bodek do { \
25445c98dacSZbigniew Bodek (void)(flags); \
25545c98dacSZbigniew Bodek mtx_unlock_spin(&(spinlock)); \
25645c98dacSZbigniew Bodek } while (0)
25745c98dacSZbigniew Bodek
25845c98dacSZbigniew Bodek
25945c98dacSZbigniew Bodek /* Wait queue related methods */
26045c98dacSZbigniew Bodek #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; }
26145c98dacSZbigniew Bodek #define ENA_WAIT_EVENT_INIT(waitqueue) \
26245c98dacSZbigniew Bodek do { \
26345c98dacSZbigniew Bodek cv_init(&((waitqueue).wq), "cv"); \
26445c98dacSZbigniew Bodek mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \
26545c98dacSZbigniew Bodek } while (0)
2669eb1615fSMarcin Wojtas #define ENA_WAIT_EVENTS_DESTROY(admin_queue) \
26745c98dacSZbigniew Bodek do { \
2689eb1615fSMarcin Wojtas struct ena_comp_ctx *comp_ctx; \
2699eb1615fSMarcin Wojtas int i; \
2709eb1615fSMarcin Wojtas for (i = 0; i < admin_queue->q_depth; i++) { \
2719eb1615fSMarcin Wojtas comp_ctx = get_comp_ctxt(admin_queue, i, false); \
2729eb1615fSMarcin Wojtas if (comp_ctx != NULL) { \
2739eb1615fSMarcin Wojtas cv_destroy(&((comp_ctx->wait_event).wq)); \
2749eb1615fSMarcin Wojtas mtx_destroy(&((comp_ctx->wait_event).mtx)); \
2759eb1615fSMarcin Wojtas } \
2769eb1615fSMarcin Wojtas } \
27745c98dacSZbigniew Bodek } while (0)
27845c98dacSZbigniew Bodek #define ENA_WAIT_EVENT_CLEAR(waitqueue) \
27945c98dacSZbigniew Bodek cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description)
28045c98dacSZbigniew Bodek #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \
28145c98dacSZbigniew Bodek do { \
28245c98dacSZbigniew Bodek mtx_lock(&((waitqueue).mtx)); \
28345c98dacSZbigniew Bodek cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \
28445c98dacSZbigniew Bodek timeout_us * hz / 1000 / 1000 ); \
28545c98dacSZbigniew Bodek mtx_unlock(&((waitqueue).mtx)); \
28645c98dacSZbigniew Bodek } while (0)
2876461d6a3SMarcin Wojtas #define ENA_WAIT_EVENT_SIGNAL(waitqueue) \
2886461d6a3SMarcin Wojtas do { \
2896461d6a3SMarcin Wojtas mtx_lock(&((waitqueue).mtx)); \
2906461d6a3SMarcin Wojtas cv_broadcast(&((waitqueue).wq)); \
2916461d6a3SMarcin Wojtas mtx_unlock(&((waitqueue).mtx)); \
2926461d6a3SMarcin Wojtas } while (0)
29345c98dacSZbigniew Bodek
29445c98dacSZbigniew Bodek #define dma_addr_t bus_addr_t
29545c98dacSZbigniew Bodek #define u8 uint8_t
29645c98dacSZbigniew Bodek #define u16 uint16_t
29745c98dacSZbigniew Bodek #define u32 uint32_t
29845c98dacSZbigniew Bodek #define u64 uint64_t
29945c98dacSZbigniew Bodek
30045c98dacSZbigniew Bodek typedef struct {
30145c98dacSZbigniew Bodek bus_addr_t paddr;
30245c98dacSZbigniew Bodek caddr_t vaddr;
30345c98dacSZbigniew Bodek bus_dma_tag_t tag;
30445c98dacSZbigniew Bodek bus_dmamap_t map;
30545c98dacSZbigniew Bodek bus_dma_segment_t seg;
30645c98dacSZbigniew Bodek int nseg;
30745c98dacSZbigniew Bodek } ena_mem_handle_t;
30845c98dacSZbigniew Bodek
30945c98dacSZbigniew Bodek struct ena_bus {
31045c98dacSZbigniew Bodek bus_space_handle_t reg_bar_h;
31145c98dacSZbigniew Bodek bus_space_tag_t reg_bar_t;
31245c98dacSZbigniew Bodek bus_space_handle_t mem_bar_h;
31345c98dacSZbigniew Bodek bus_space_tag_t mem_bar_t;
31445c98dacSZbigniew Bodek };
31545c98dacSZbigniew Bodek
31645c98dacSZbigniew Bodek typedef uint32_t ena_atomic32_t;
31745c98dacSZbigniew Bodek
3188483b844SMarcin Wojtas #define ENA_PRIu64 PRIu64
3198483b844SMarcin Wojtas
3208483b844SMarcin Wojtas typedef uint64_t ena_time_t;
321adfed2d8SArthur Kiyanovski typedef uint64_t ena_time_high_res_t;
3229eb1615fSMarcin Wojtas typedef struct ifnet ena_netdev;
3238483b844SMarcin Wojtas
32445c98dacSZbigniew Bodek void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
32545c98dacSZbigniew Bodek int error);
32645c98dacSZbigniew Bodek int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
327eb4c4f4aSMarcin Wojtas int mapflags, bus_size_t alignment, int domain);
32845c98dacSZbigniew Bodek
3298483b844SMarcin Wojtas static inline uint32_t
ena_reg_read32(struct ena_bus * bus,bus_size_t offset)3308483b844SMarcin Wojtas ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
3318483b844SMarcin Wojtas {
3328483b844SMarcin Wojtas uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset);
3338483b844SMarcin Wojtas rmb();
3348483b844SMarcin Wojtas return v;
3358483b844SMarcin Wojtas }
3368483b844SMarcin Wojtas
337a195fab0SMarcin Wojtas #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \
338a195fab0SMarcin Wojtas do { \
339a195fab0SMarcin Wojtas int count, i; \
340a195fab0SMarcin Wojtas volatile uint64_t *to = (volatile uint64_t *)(dst); \
341a195fab0SMarcin Wojtas const uint64_t *from = (const uint64_t *)(src); \
342a195fab0SMarcin Wojtas count = (size) / 8; \
343a195fab0SMarcin Wojtas \
344a195fab0SMarcin Wojtas for (i = 0; i < count; i++, from++, to++) \
345a195fab0SMarcin Wojtas *to = *from; \
346a195fab0SMarcin Wojtas } while (0)
347a195fab0SMarcin Wojtas
34845c98dacSZbigniew Bodek #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
349eb4c4f4aSMarcin Wojtas
350eb4c4f4aSMarcin Wojtas #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \
351eb4c4f4aSMarcin Wojtas do { \
352eb4c4f4aSMarcin Wojtas (virt) = malloc_domainset((size), M_DEVBUF, \
353eb4c4f4aSMarcin Wojtas (node) < 0 ? DOMAINSET_RR() : DOMAINSET_PREF(node), \
354eb4c4f4aSMarcin Wojtas M_NOWAIT | M_ZERO); \
355eb4c4f4aSMarcin Wojtas (void)(dev_node); \
356eb4c4f4aSMarcin Wojtas } while (0)
357eb4c4f4aSMarcin Wojtas
3588483b844SMarcin Wojtas #define ENA_MEM_FREE(dmadev, ptr, size) \
3598483b844SMarcin Wojtas do { \
3608483b844SMarcin Wojtas (void)(size); \
3618483b844SMarcin Wojtas free(ptr, M_DEVBUF); \
3628483b844SMarcin Wojtas } while (0)
3634f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \
364eb4c4f4aSMarcin Wojtas dma, node, dev_node, alignment) \
36545c98dacSZbigniew Bodek do { \
366eb4c4f4aSMarcin Wojtas ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \
367eb4c4f4aSMarcin Wojtas (node)); \
368eb4c4f4aSMarcin Wojtas (virt) = (void *)(dma).vaddr; \
369eb4c4f4aSMarcin Wojtas (phys) = (dma).paddr; \
37045c98dacSZbigniew Bodek (void)(dev_node); \
37145c98dacSZbigniew Bodek } while (0)
37245c98dacSZbigniew Bodek
3734f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, \
3744f8f476eSMarcin Wojtas node, dev_node) \
3754f8f476eSMarcin Wojtas ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, \
3764f8f476eSMarcin Wojtas phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT)
3774f8f476eSMarcin Wojtas
3784f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma, \
3794f8f476eSMarcin Wojtas alignment) \
38045c98dacSZbigniew Bodek do { \
381eb4c4f4aSMarcin Wojtas ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \
382eb4c4f4aSMarcin Wojtas ENA_NODE_ANY); \
38345c98dacSZbigniew Bodek (virt) = (void *)(dma).vaddr; \
38445c98dacSZbigniew Bodek (phys) = (dma).paddr; \
38545c98dacSZbigniew Bodek } while (0)
38645c98dacSZbigniew Bodek
3874f8f476eSMarcin Wojtas #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \
3884f8f476eSMarcin Wojtas ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, \
3894f8f476eSMarcin Wojtas phys, dma, DEFAULT_ALLOC_ALIGNMENT)
3904f8f476eSMarcin Wojtas
39145c98dacSZbigniew Bodek #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \
39245c98dacSZbigniew Bodek do { \
39345c98dacSZbigniew Bodek (void)size; \
39445c98dacSZbigniew Bodek bus_dmamap_unload((dma).tag, (dma).map); \
39545c98dacSZbigniew Bodek bus_dmamem_free((dma).tag, (virt), (dma).map); \
39645c98dacSZbigniew Bodek bus_dma_tag_destroy((dma).tag); \
39745c98dacSZbigniew Bodek (dma).tag = NULL; \
39845c98dacSZbigniew Bodek (virt) = NULL; \
39945c98dacSZbigniew Bodek } while (0)
40045c98dacSZbigniew Bodek
40145c98dacSZbigniew Bodek /* Register R/W methods */
40245c98dacSZbigniew Bodek #define ENA_REG_WRITE32(bus, value, offset) \
4038483b844SMarcin Wojtas do { \
4048483b844SMarcin Wojtas wmb(); \
4058483b844SMarcin Wojtas ENA_REG_WRITE32_RELAXED(bus, value, offset); \
4068483b844SMarcin Wojtas } while (0)
4078483b844SMarcin Wojtas
4088483b844SMarcin Wojtas #define ENA_REG_WRITE32_RELAXED(bus, value, offset) \
40945c98dacSZbigniew Bodek bus_space_write_4( \
41045c98dacSZbigniew Bodek ((struct ena_bus*)bus)->reg_bar_t, \
41145c98dacSZbigniew Bodek ((struct ena_bus*)bus)->reg_bar_h, \
41245c98dacSZbigniew Bodek (bus_size_t)(offset), (value))
41345c98dacSZbigniew Bodek
41445c98dacSZbigniew Bodek #define ENA_REG_READ32(bus, offset) \
4158483b844SMarcin Wojtas ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset))
41645c98dacSZbigniew Bodek
41767ec48bbSMarcin Wojtas #define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync( \
41867ec48bbSMarcin Wojtas (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE)
41967ec48bbSMarcin Wojtas #define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync( \
42067ec48bbSMarcin Wojtas (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD)
42167ec48bbSMarcin Wojtas #define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync( \
42267ec48bbSMarcin Wojtas (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD)
42367ec48bbSMarcin Wojtas #define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle)
424a195fab0SMarcin Wojtas
42545c98dacSZbigniew Bodek #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
42645c98dacSZbigniew Bodek
42745c98dacSZbigniew Bodek #define VLAN_HLEN sizeof(struct ether_vlan_header)
42845c98dacSZbigniew Bodek
42967ec48bbSMarcin Wojtas #define prefetch(x) (void)(x)
43067ec48bbSMarcin Wojtas #define prefetchw(x) (void)(x)
43145c98dacSZbigniew Bodek
43245c98dacSZbigniew Bodek /* DMA buffers access */
43345c98dacSZbigniew Bodek #define dma_unmap_addr(p, name) ((p)->dma->name)
43445c98dacSZbigniew Bodek #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v))
43545c98dacSZbigniew Bodek #define dma_unmap_len(p, name) ((p)->name)
43645c98dacSZbigniew Bodek #define dma_unmap_len_set(p, name, v) (((p)->name) = (v))
43745c98dacSZbigniew Bodek
43845c98dacSZbigniew Bodek #define memcpy_toio memcpy
43945c98dacSZbigniew Bodek
44045c98dacSZbigniew Bodek #define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1)
44145c98dacSZbigniew Bodek #define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1)
44245c98dacSZbigniew Bodek #define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR)
44345c98dacSZbigniew Bodek #define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL)
44445c98dacSZbigniew Bodek
44545c98dacSZbigniew Bodek #define barrier() __asm__ __volatile__("": : :"memory")
44667ec48bbSMarcin Wojtas #define dma_rmb() barrier()
44767ec48bbSMarcin Wojtas #define mmiowb() barrier()
44867ec48bbSMarcin Wojtas
44945c98dacSZbigniew Bodek #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x))
45045c98dacSZbigniew Bodek #define READ_ONCE(x) ({ \
45145c98dacSZbigniew Bodek __typeof(x) __var; \
45245c98dacSZbigniew Bodek barrier(); \
45345c98dacSZbigniew Bodek __var = ACCESS_ONCE(x); \
45445c98dacSZbigniew Bodek barrier(); \
45545c98dacSZbigniew Bodek __var; \
45645c98dacSZbigniew Bodek })
45767ec48bbSMarcin Wojtas #define READ_ONCE8(x) READ_ONCE(x)
45867ec48bbSMarcin Wojtas #define READ_ONCE16(x) READ_ONCE(x)
45967ec48bbSMarcin Wojtas #define READ_ONCE32(x) READ_ONCE(x)
46067ec48bbSMarcin Wojtas
46167ec48bbSMarcin Wojtas #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
46267ec48bbSMarcin Wojtas #define lower_32_bits(n) ((uint32_t)(n))
46367ec48bbSMarcin Wojtas
46467ec48bbSMarcin Wojtas #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
46545c98dacSZbigniew Bodek
4668483b844SMarcin Wojtas #define ENA_FFS(x) ffs(x)
4678483b844SMarcin Wojtas
4688483b844SMarcin Wojtas void ena_rss_key_fill(void *key, size_t size);
4698483b844SMarcin Wojtas
4708483b844SMarcin Wojtas #define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size)
4718483b844SMarcin Wojtas
472a195fab0SMarcin Wojtas #include "ena_defs/ena_includes.h"
47345c98dacSZbigniew Bodek
474adfed2d8SArthur Kiyanovski #define ENA_BITS_PER_U64(bitmap) (bitcount64(bitmap))
475adfed2d8SArthur Kiyanovski
47645c98dacSZbigniew Bodek #endif /* ENA_PLAT_H_ */
477