1 /*-
2 * BSD LICENSE
3 *
4 * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #ifndef ENA_PLAT_H_
35 #define ENA_PLAT_H_
36
37 #include <sys/cdefs.h>
38 #if 0
39 __FBSDID("$FreeBSD: head/sys/contrib/ena-com/ena_plat.h 333453 2018-05-10 09:25:51Z mw $");
40 #endif
41 __KERNEL_RCSID(0, "$NetBSD: ena_plat.h,v 1.9 2022/04/09 23:44:54 riastradh Exp $");
42
43 #include <sys/param.h>
44 #include <sys/systm.h>
45
46 #include <sys/bus.h>
47 #include <sys/condvar.h>
48 #include <sys/endian.h>
49 #include <sys/kernel.h>
50 #include <sys/kthread.h>
51 #include <sys/malloc.h>
52 #include <sys/mbuf.h>
53 #include <sys/module.h>
54 #include <sys/proc.h>
55 #include <sys/socket.h>
56 #include <sys/sockio.h>
57 #include <sys/sysctl.h>
58 #include <sys/types.h>
59 #include <sys/bus.h>
60 #include <sys/atomic.h>
61
62 #include <net/if.h>
63 #include <net/if_dl.h>
64 #include <net/if_media.h>
65 #include <net/if_ether.h>
66
67 #include <net/bpf.h>
68
69 #include <net/rss_config.h>
70
71 #include <netinet/in.h> /* XXX for struct ip */
72 #include <netinet/in_systm.h> /* XXX for struct ip */
73 #include <netinet/ip.h> /* XXX for struct ip */
74 #include <netinet/ip6.h> /* XXX for struct ip6_hdr */
75 #include <netinet/tcp.h> /* XXX for struct tcphdr */
76
77 #include <dev/pci/pcivar.h>
78 #include <dev/pci/pcireg.h>
79
80 extern struct ena_bus_space ebs;
81
82 /* Levels */
83 #define ENA_ALERT (1 << 0) /* Alerts are providing more error info. */
84 #define ENA_WARNING (1 << 1) /* Driver output is more error sensitive. */
85 #define ENA_INFO (1 << 2) /* Provides additional driver info. */
86 #define ENA_DBG (1 << 3) /* Driver output for debugging. */
87 /* Detailed info that will be printed with ENA_INFO or ENA_DEBUG flag. */
88 #define ENA_TXPTH (1 << 4) /* Allows TX path tracing. */
89 #define ENA_RXPTH (1 << 5) /* Allows RX path tracing. */
90 #define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */
91 #define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */
92 #define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */
93
94 extern int ena_log_level;
95
96 #define ena_trace_raw(level, fmt, args...) \
97 do { \
98 if (((level) & ena_log_level) != (level)) \
99 break; \
100 printf(fmt, ##args); \
101 } while (0)
102
103 #define ena_trace(level, fmt, args...) \
104 ena_trace_raw(level, "%s() [LID:%d]: " \
105 fmt " \n", __func__, curlwp->l_lid, ##args)
106
107
108 #define ena_trc_dbg(format, arg...) ena_trace(ENA_DBG, format, ##arg)
109 #define ena_trc_info(format, arg...) ena_trace(ENA_INFO, format, ##arg)
110 #define ena_trc_warn(format, arg...) ena_trace(ENA_WARNING, format, ##arg)
111 #define ena_trc_err(format, arg...) ena_trace(ENA_ALERT, format, ##arg)
112
113 #define unlikely(x) __predict_false(x)
114 #define likely(x) __predict_true(x)
115
116 #define __iomem
117 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE)
118
119 #define MAX_ERRNO 4095
120 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
121
122 #define ENA_ASSERT(cond, format, arg...) \
123 do { \
124 if (unlikely(!(cond))) { \
125 ena_trc_err( \
126 "Assert failed on %s:%s:%d:" format, \
127 __FILE__, __func__, __LINE__, ##arg); \
128 } \
129 } while (0)
130
131 #define ENA_WARN(cond, format, arg...) \
132 do { \
133 if (unlikely((cond))) { \
134 ena_trc_warn(format, ##arg); \
135 } \
136 } while (0)
137
IS_ERR(const void * ptr)138 static inline long IS_ERR(const void *ptr)
139 {
140 return IS_ERR_VALUE((unsigned long)ptr);
141 }
142
ERR_PTR(long error)143 static inline void *ERR_PTR(long error)
144 {
145 return (void *)error;
146 }
147
PTR_ERR(const void * ptr)148 static inline long PTR_ERR(const void *ptr)
149 {
150 return (long) ptr;
151 }
152
153 #define GENMASK(h, l) (((1U << ((h) - (l) + 1)) - 1) << (l))
154 #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
155 #define BIT(x) (1UL << (x))
156
157 #define ENA_ABORT() BUG()
158 #define BUG() panic("ENA BUG")
159
160 #define SZ_256 (256)
161 #define SZ_4K (4096)
162
163 #define ENA_COM_OK 0
164 #define ENA_COM_FAULT EFAULT
165 #define ENA_COM_INVAL EINVAL
166 #define ENA_COM_NO_MEM ENOMEM
167 #define ENA_COM_NO_SPACE ENOSPC
168 #define ENA_COM_TRY_AGAIN -1
169 #define ENA_COM_UNSUPPORTED EOPNOTSUPP
170 #define ENA_COM_NO_DEVICE ENODEV
171 #define ENA_COM_PERMISSION EPERM
172 #define ENA_COM_TIMER_EXPIRED ETIMEDOUT
173
174 static inline int
ENA_MSLEEP(int x)175 ENA_MSLEEP(int x)
176 {
177 if (cold) {
178 while (x >= 1000000) {
179 delay(1000000);
180 x -= 1000000;
181 }
182 if (x > 0)
183 delay(x);
184 return EWOULDBLOCK;
185 } else {
186 return kpause("enaw", false, mstohz(x), NULL);
187 }
188 }
189
190 #define ENA_UDELAY(x) DELAY(x)
191 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \
192 mstohz(timeout_us * (1000 / 100)) /* XXX assumes 100 ms sleep */
193 #define ENA_TIME_EXPIRE(timeout) ((timeout)-- <= 0)
194 #define ENA_MIGHT_SLEEP()
195
196 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
197 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y))
198
199 #define ENA_MIN32(x,y) MIN(x, y)
200 #define ENA_MIN16(x,y) MIN(x, y)
201 #define ENA_MIN8(x,y) MIN(x, y)
202
203 #define ENA_MAX32(x,y) MAX(x, y)
204 #define ENA_MAX16(x,y) MAX(x, y)
205 #define ENA_MAX8(x,y) MAX(x, y)
206
207 /* Spinlock related methods */
208 #define ena_spinlock_t kmutex_t
209 #define ENA_SPINLOCK_INIT(spinlock) \
210 mutex_init(&(spinlock), MUTEX_DEFAULT, IPL_NET)
211 #define ENA_SPINLOCK_DESTROY(spinlock) \
212 do { \
213 mutex_destroy(&(spinlock)); \
214 } while (0)
215 #define ENA_SPINLOCK_LOCK(spinlock, flags) \
216 do { \
217 (void)(flags); \
218 mutex_enter(&(spinlock)); \
219 } while (0)
220 #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \
221 do { \
222 (void)(flags); \
223 mutex_exit(&(spinlock)); \
224 } while (0)
225
226
227 /* Wait queue related methods */
228 #define ena_wait_event_t struct { kcondvar_t wq; kmutex_t mtx; }
229 #define ENA_WAIT_EVENT_INIT(waitqueue) \
230 do { \
231 cv_init(&((waitqueue).wq), "enacv"); \
232 mutex_init(&((waitqueue).mtx), MUTEX_DEFAULT, IPL_NET); \
233 } while (0)
234 #define ENA_WAIT_EVENT_DESTROY(waitqueue) \
235 do { \
236 cv_destroy(&((waitqueue).wq)); \
237 mutex_destroy(&((waitqueue).mtx)); \
238 } while (0)
239 #define ENA_WAIT_EVENT_CLEAR(waitqueue) \
240 cv_init(&((waitqueue).wq), "enacv")
241 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \
242 do { \
243 mutex_enter(&((waitqueue).mtx)); \
244 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \
245 timeout_us * hz / 1000 / 1000 ); \
246 mutex_exit(&((waitqueue).mtx)); \
247 } while (0)
248 #define ENA_WAIT_EVENT_SIGNAL(waitqueue) \
249 do { \
250 mutex_enter(&((waitqueue).mtx)); \
251 cv_broadcast(&((waitqueue).wq)); \
252 mutex_exit(&((waitqueue).mtx)); \
253 } while (0)
254
255 #define dma_addr_t bus_addr_t
256 #define u8 uint8_t
257 #define u16 uint16_t
258 #define u32 uint32_t
259 #define u64 uint64_t
260
261 typedef struct {
262 paddr_t paddr;
263 void *vaddr;
264 bus_dma_tag_t tag;
265 bus_dmamap_t map;
266 bus_dma_segment_t seg;
267 int nseg;
268 } ena_mem_handle_t;
269
270 struct ena_bus {
271 bus_space_handle_t reg_bar_h;
272 bus_space_tag_t reg_bar_t;
273 bus_space_handle_t mem_bar_h;
274 bus_space_tag_t mem_bar_t;
275 };
276
277 typedef uint32_t ena_atomic32_t;
278
279 void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
280 int error);
281 int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
282 int mapflags);
283
284 #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \
285 do { \
286 int count, i; \
287 volatile uint64_t *to = (volatile uint64_t *)(dst); \
288 const uint64_t *from = (const uint64_t *)(src); \
289 count = (size) / 8; \
290 \
291 for (i = 0; i < count; i++, from++, to++) \
292 *to = *from; \
293 } while (0)
294
295 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
296 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL)
297 #define ENA_MEM_FREE(dmadev, ptr) free(ptr, M_DEVBUF)
298 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, node, \
299 dev_node) \
300 do { \
301 ((virt) = NULL); \
302 (void)(dev_node); \
303 } while (0)
304
305 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \
306 do { \
307 ena_dma_alloc((dmadev), (size), &(dma), 0); \
308 (virt) = (void *)(dma).vaddr; \
309 (phys) = (dma).paddr; \
310 } while (0)
311
312 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \
313 do { \
314 (void)size; \
315 size_t mapsize = (dma).map->dm_mapsize; \
316 bus_dmamap_unload((dma).tag, (dma).map); \
317 bus_dmamem_unmap((dma).tag, (dma).vaddr, mapsize); \
318 bus_dmamem_free((dma).tag, &(dma).seg, (dma).nseg); \
319 bus_dmamap_destroy((dma).tag, (dma).map); \
320 (dma).tag = NULL; \
321 (virt) = NULL; \
322 } while (0)
323
324 /* Register R/W methods */
325 #define ENA_REG_WRITE32(bus, value, offset) \
326 bus_space_write_4( \
327 ((struct ena_bus*)bus)->reg_bar_t, \
328 ((struct ena_bus*)bus)->reg_bar_h, \
329 (bus_size_t)(offset), (value))
330
331 #define ENA_REG_READ32(bus, offset) \
332 bus_space_read_4( \
333 ((struct ena_bus*)bus)->reg_bar_t, \
334 ((struct ena_bus*)bus)->reg_bar_h, \
335 (bus_size_t)(offset))
336
337 #define ENA_DB_SYNC(mem_handle) bus_dmamap_sync((mem_handle)->tag, \
338 (mem_handle)->map, 0, (mem_handle)->map->dm_mapsize, \
339 BUS_DMASYNC_PREREAD)
340
341 #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
342
343 #define VLAN_HLEN sizeof(struct ether_vlan_header)
344 #define CSUM_OFFLOAD (M_CSUM_IPv4|M_CSUM_TCPv4|M_CSUM_UDPv4)
345
346 #if defined(__i386__) || defined(__amd64__)
347 static __inline
prefetch(void * x)348 void prefetch(void *x)
349 {
350 __asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
351 }
352 #else
353 #define prefetch(x)
354 #endif
355
356 /* DMA buffers access */
357 #define dma_unmap_addr(p, name) ((p)->dma->name)
358 #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v))
359 #define dma_unmap_len(p, name) ((p)->name)
360 #define dma_unmap_len_set(p, name, v) (((p)->name) = (v))
361
362 #define memcpy_toio memcpy
363
364 #define ATOMIC32_INC(I32_PTR) atomic_inc_32(I32_PTR)
365 #define ATOMIC32_DEC(I32_PTR) atomic_dec_32(I32_PTR)
366 #define ATOMIC32_READ(I32_PTR) atomic_cas_32(I32_PTR, 0, 0)
367 #define ATOMIC32_SET(I32_PTR, VAL) atomic_swap_32(I32_PTR, VAL)
368
369 #define barrier() __asm__ __volatile__("": : :"memory")
370 #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x))
371 #define READ_ONCE(x) ({ \
372 __typeof(x) __var; \
373 barrier(); \
374 __var = ACCESS_ONCE(x); \
375 barrier(); \
376 __var; \
377 })
378
379 #include "ena_defs/ena_includes.h"
380
381 /*
382 * XXX This is not really right. Need to adjust the driver to use
383 * bus_space_barrier or bus_dmamap_sync.
384 */
385 #if defined(__i386__) || defined(__x86_64__)
386 #include <x86/cpufunc.h>
387 #define rmb() x86_lfence()
388 #define wmb() x86_sfence()
389 #define mb() x86_mfence()
390 #elif defined(__aarch64__)
391 #define rmb() __asm __volatile("dsb ld" ::: "memory")
392 #define wmb() __asm __volatile("dsb st" ::: "memory")
393 #define mb() __asm __volatile("dsb sy" ::: "memory")
394 #else
395 #define rmb() membar_acquire()
396 #define wmb() membar_release()
397 #define mb() membar_sync()
398 #endif
399
400 #endif /* ENA_PLAT_H_ */
401