1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2015-2023 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #ifndef ENA_PLAT_H_
35 #define ENA_PLAT_H_
36
37 #include <sys/cdefs.h>
38 __FBSDID("$FreeBSD$");
39
40 #include <sys/param.h>
41 #include <sys/systm.h>
42
43 #include <sys/bus.h>
44 #include <sys/condvar.h>
45 #include <sys/domainset.h>
46 #include <sys/endian.h>
47 #include <sys/kernel.h>
48 #include <sys/kthread.h>
49 #include <sys/malloc.h>
50 #include <sys/mbuf.h>
51 #include <sys/module.h>
52 #include <sys/rman.h>
53 #include <sys/proc.h>
54 #include <sys/smp.h>
55 #include <sys/socket.h>
56 #include <sys/sockio.h>
57 #include <sys/sysctl.h>
58 #include <sys/taskqueue.h>
59 #include <sys/eventhandler.h>
60 #include <sys/types.h>
61 #include <sys/timetc.h>
62 #include <sys/cdefs.h>
63
64 #include <machine/atomic.h>
65 #include <machine/bus.h>
66 #include <machine/in_cksum.h>
67 #include <machine/pcpu.h>
68 #include <machine/resource.h>
69 #include <machine/_inttypes.h>
70
71 #include <net/bpf.h>
72 #include <net/ethernet.h>
73 #include <net/if.h>
74 #include <net/if_var.h>
75 #include <net/if_arp.h>
76 #include <net/if_dl.h>
77 #include <net/if_media.h>
78
79 #include <net/if_types.h>
80 #include <net/if_vlan_var.h>
81
82 #include <netinet/in_systm.h>
83 #include <netinet/in.h>
84 #include <netinet/if_ether.h>
85 #include <netinet/ip.h>
86 #include <netinet/ip6.h>
87 #include <netinet/tcp.h>
88 #include <netinet/tcp_lro.h>
89 #include <netinet/udp.h>
90
91 #include <dev/led/led.h>
92 #include <dev/pci/pcivar.h>
93 #include <dev/pci/pcireg.h>
94
95 enum ena_log_t {
96 ENA_ERR = 0,
97 ENA_WARN,
98 ENA_INFO,
99 ENA_DBG,
100 };
101
102 extern int ena_log_level;
103
104 #define ena_log(dev, level, fmt, args...) \
105 do { \
106 if (ENA_ ## level <= ena_log_level) \
107 device_printf((dev), fmt, ##args); \
108 } while (0)
109
110 #define ena_log_raw(level, fmt, args...) \
111 do { \
112 if (ENA_ ## level <= ena_log_level) \
113 printf(fmt, ##args); \
114 } while (0)
115
116 #define ena_log_unused(dev, level, fmt, args...) \
117 do { \
118 (void)(dev); \
119 } while (0)
120
121 #ifdef ENA_LOG_IO_ENABLE
122 #define ena_log_io(dev, level, fmt, args...) \
123 ena_log((dev), level, fmt, ##args)
124 #else
125 #define ena_log_io(dev, level, fmt, args...) \
126 ena_log_unused((dev), level, fmt, ##args)
127 #endif
128
129 #define ena_log_nm(dev, level, fmt, args...) \
130 ena_log((dev), level, "[nm] " fmt, ##args)
131
132 extern struct ena_bus_space ebs;
133
134 #define DEFAULT_ALLOC_ALIGNMENT 8
135 #define ENA_CDESC_RING_SIZE_ALIGNMENT (1 << 12) /* 4K */
136
137 #define container_of(ptr, type, member) \
138 ({ \
139 const __typeof(((type *)0)->member) *__p = (ptr); \
140 (type *)((uintptr_t)__p - offsetof(type, member)); \
141 })
142
143 #define ena_trace(ctx, level, fmt, args...) \
144 ena_log((ctx)->dmadev, level, "%s() [TID:%d]: " \
145 fmt, __func__, curthread->td_tid, ##args)
146
147 #define ena_trc_dbg(ctx, format, arg...) \
148 ena_trace(ctx, DBG, format, ##arg)
149 #define ena_trc_info(ctx, format, arg...) \
150 ena_trace(ctx, INFO, format, ##arg)
151 #define ena_trc_warn(ctx, format, arg...) \
152 ena_trace(ctx, WARN, format, ##arg)
153 #define ena_trc_err(ctx, format, arg...) \
154 ena_trace(ctx, ERR, format, ##arg)
155
156 #define unlikely(x) __predict_false(!!(x))
157 #define likely(x) __predict_true(!!(x))
158
159 #define __iomem
160 #define ____cacheline_aligned __aligned(CACHE_LINE_SIZE)
161
162 #define MAX_ERRNO 4095
163 #define IS_ERR_VALUE(x) unlikely((x) <= (unsigned long)MAX_ERRNO)
164
165 #define ENA_WARN(cond, ctx, format, arg...) \
166 do { \
167 if (unlikely((cond))) { \
168 ena_trc_warn(ctx, format, ##arg); \
169 } \
170 } while (0)
171
IS_ERR(const void * ptr)172 static inline long IS_ERR(const void *ptr)
173 {
174 return IS_ERR_VALUE((unsigned long)ptr);
175 }
176
ERR_PTR(long error)177 static inline void *ERR_PTR(long error)
178 {
179 return (void *)error;
180 }
181
PTR_ERR(const void * ptr)182 static inline long PTR_ERR(const void *ptr)
183 {
184 return (long) ptr;
185 }
186
187 #define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h))))
188 #define GENMASK_ULL(h, l) (((~0ULL) << (l)) & (~0ULL >> (64 - 1 - (h))))
189 #define BIT(x) (1UL << (x))
190 #define BIT64(x) BIT(x)
191 #define ENA_ABORT() BUG()
192 #define BUG() panic("ENA BUG")
193
194 #define SZ_256 (256)
195 #define SZ_4K (4096)
196
197 #define ENA_COM_OK 0
198 #define ENA_COM_FAULT EFAULT
199 #define ENA_COM_INVAL EINVAL
200 #define ENA_COM_NO_MEM ENOMEM
201 #define ENA_COM_NO_SPACE ENOSPC
202 #define ENA_COM_TRY_AGAIN -1
203 #define ENA_COM_UNSUPPORTED EOPNOTSUPP
204 #define ENA_COM_NO_DEVICE ENODEV
205 #define ENA_COM_PERMISSION EPERM
206 #define ENA_COM_TIMER_EXPIRED ETIMEDOUT
207 #define ENA_COM_EIO EIO
208 #define ENA_COM_DEVICE_BUSY EBUSY
209
210 #define ENA_NODE_ANY (-1)
211
212 #define ENA_MSLEEP(x) pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
213 #define ENA_USLEEP(x) pause_sbt("ena", SBT_1US * (x), SBT_1US, 0)
214 #define ENA_UDELAY(x) DELAY(x)
215 #define ENA_GET_SYSTEM_TIMEOUT(timeout_us) \
216 ((long)cputick2usec(cpu_ticks()) + (timeout_us))
217 #define ENA_TIME_EXPIRE(timeout) ((timeout) < cputick2usec(cpu_ticks()))
218 #define ENA_TIME_EXPIRE_HIGH_RES ENA_TIME_EXPIRE
219 #define ENA_TIME_INIT_HIGH_RES() (0)
220 #define ENA_TIME_COMPARE_HIGH_RES(time1, time2) \
221 ((time1 < time2) ? -1 : ((time1 > time2) ? 1 : 0))
222 #define ENA_GET_SYSTEM_TIMEOUT_HIGH_RES(current_time, timeout_us) \
223 ((long)cputick2usec(cpu_ticks()) + (timeout_us))
224 #define ENA_GET_SYSTEM_TIME_HIGH_RES() ENA_GET_SYSTEM_TIMEOUT(0)
225 #define ENA_MIGHT_SLEEP()
226
227 #define min_t(type, _x, _y) ((type)(_x) < (type)(_y) ? (type)(_x) : (type)(_y))
228 #define max_t(type, _x, _y) ((type)(_x) > (type)(_y) ? (type)(_x) : (type)(_y))
229
230 #define ENA_MIN32(x,y) MIN(x, y)
231 #define ENA_MIN16(x,y) MIN(x, y)
232 #define ENA_MIN8(x,y) MIN(x, y)
233
234 #define ENA_MAX32(x,y) MAX(x, y)
235 #define ENA_MAX16(x,y) MAX(x, y)
236 #define ENA_MAX8(x,y) MAX(x, y)
237
238 /* Spinlock related methods */
239 #define ena_spinlock_t struct mtx
240 #define ENA_SPINLOCK_INIT(spinlock) \
241 mtx_init(&(spinlock), "ena_spin", NULL, MTX_SPIN)
242 #define ENA_SPINLOCK_DESTROY(spinlock) \
243 do { \
244 if (mtx_initialized(&(spinlock))) \
245 mtx_destroy(&(spinlock)); \
246 } while (0)
247 #define ENA_SPINLOCK_LOCK(spinlock, flags) \
248 do { \
249 (void)(flags); \
250 mtx_lock_spin(&(spinlock)); \
251 } while (0)
252 #define ENA_SPINLOCK_UNLOCK(spinlock, flags) \
253 do { \
254 (void)(flags); \
255 mtx_unlock_spin(&(spinlock)); \
256 } while (0)
257
258
259 /* Wait queue related methods */
260 #define ena_wait_event_t struct { struct cv wq; struct mtx mtx; }
261 #define ENA_WAIT_EVENT_INIT(waitqueue) \
262 do { \
263 cv_init(&((waitqueue).wq), "cv"); \
264 mtx_init(&((waitqueue).mtx), "wq", NULL, MTX_DEF); \
265 } while (0)
266 #define ENA_WAIT_EVENTS_DESTROY(admin_queue) \
267 do { \
268 struct ena_comp_ctx *comp_ctx; \
269 int i; \
270 for (i = 0; i < admin_queue->q_depth; i++) { \
271 comp_ctx = get_comp_ctxt(admin_queue, i, false); \
272 if (comp_ctx != NULL) { \
273 cv_destroy(&((comp_ctx->wait_event).wq)); \
274 mtx_destroy(&((comp_ctx->wait_event).mtx)); \
275 } \
276 } \
277 } while (0)
278 #define ENA_WAIT_EVENT_CLEAR(waitqueue) \
279 cv_init(&((waitqueue).wq), (waitqueue).wq.cv_description)
280 #define ENA_WAIT_EVENT_WAIT(waitqueue, timeout_us) \
281 do { \
282 mtx_lock(&((waitqueue).mtx)); \
283 cv_timedwait(&((waitqueue).wq), &((waitqueue).mtx), \
284 timeout_us * hz / 1000 / 1000 ); \
285 mtx_unlock(&((waitqueue).mtx)); \
286 } while (0)
287 #define ENA_WAIT_EVENT_SIGNAL(waitqueue) \
288 do { \
289 mtx_lock(&((waitqueue).mtx)); \
290 cv_broadcast(&((waitqueue).wq)); \
291 mtx_unlock(&((waitqueue).mtx)); \
292 } while (0)
293
294 #define dma_addr_t bus_addr_t
295 #define u8 uint8_t
296 #define u16 uint16_t
297 #define u32 uint32_t
298 #define u64 uint64_t
299
300 typedef struct {
301 bus_addr_t paddr;
302 caddr_t vaddr;
303 bus_dma_tag_t tag;
304 bus_dmamap_t map;
305 bus_dma_segment_t seg;
306 int nseg;
307 } ena_mem_handle_t;
308
309 struct ena_bus {
310 bus_space_handle_t reg_bar_h;
311 bus_space_tag_t reg_bar_t;
312 bus_space_handle_t mem_bar_h;
313 bus_space_tag_t mem_bar_t;
314 };
315
316 typedef uint32_t ena_atomic32_t;
317
318 #define ENA_PRIu64 PRIu64
319
320 typedef uint64_t ena_time_t;
321 typedef uint64_t ena_time_high_res_t;
322 typedef struct ifnet ena_netdev;
323
324 void ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
325 int error);
326 int ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
327 int mapflags, bus_size_t alignment, int domain);
328
329 static inline uint32_t
ena_reg_read32(struct ena_bus * bus,bus_size_t offset)330 ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
331 {
332 uint32_t v = bus_space_read_4(bus->reg_bar_t, bus->reg_bar_h, offset);
333 rmb();
334 return v;
335 }
336
337 #define ENA_MEMCPY_TO_DEVICE_64(dst, src, size) \
338 do { \
339 int count, i; \
340 volatile uint64_t *to = (volatile uint64_t *)(dst); \
341 const uint64_t *from = (const uint64_t *)(src); \
342 count = (size) / 8; \
343 \
344 for (i = 0; i < count; i++, from++, to++) \
345 *to = *from; \
346 } while (0)
347
348 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
349
350 #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \
351 do { \
352 (virt) = malloc_domainset((size), M_DEVBUF, \
353 (node) < 0 ? DOMAINSET_RR() : DOMAINSET_PREF(node), \
354 M_NOWAIT | M_ZERO); \
355 (void)(dev_node); \
356 } while (0)
357
358 #define ENA_MEM_FREE(dmadev, ptr, size) \
359 do { \
360 (void)(size); \
361 free(ptr, M_DEVBUF); \
362 } while (0)
363 #define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys, \
364 dma, node, dev_node, alignment) \
365 do { \
366 ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \
367 (node)); \
368 (virt) = (void *)(dma).vaddr; \
369 (phys) = (dma).paddr; \
370 (void)(dev_node); \
371 } while (0)
372
373 #define ENA_MEM_ALLOC_COHERENT_NODE(dmadev, size, virt, phys, handle, \
374 node, dev_node) \
375 ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, \
376 phys, handle, node, dev_node, DEFAULT_ALLOC_ALIGNMENT)
377
378 #define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma, \
379 alignment) \
380 do { \
381 ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment), \
382 ENA_NODE_ANY); \
383 (virt) = (void *)(dma).vaddr; \
384 (phys) = (dma).paddr; \
385 } while (0)
386
387 #define ENA_MEM_ALLOC_COHERENT(dmadev, size, virt, phys, dma) \
388 ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, \
389 phys, dma, DEFAULT_ALLOC_ALIGNMENT)
390
391 #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, dma) \
392 do { \
393 (void)size; \
394 bus_dmamap_unload((dma).tag, (dma).map); \
395 bus_dmamem_free((dma).tag, (virt), (dma).map); \
396 bus_dma_tag_destroy((dma).tag); \
397 (dma).tag = NULL; \
398 (virt) = NULL; \
399 } while (0)
400
401 /* Register R/W methods */
402 #define ENA_REG_WRITE32(bus, value, offset) \
403 do { \
404 wmb(); \
405 ENA_REG_WRITE32_RELAXED(bus, value, offset); \
406 } while (0)
407
408 #define ENA_REG_WRITE32_RELAXED(bus, value, offset) \
409 bus_space_write_4( \
410 ((struct ena_bus*)bus)->reg_bar_t, \
411 ((struct ena_bus*)bus)->reg_bar_h, \
412 (bus_size_t)(offset), (value))
413
414 #define ENA_REG_READ32(bus, offset) \
415 ena_reg_read32((struct ena_bus*)(bus), (bus_size_t)(offset))
416
417 #define ENA_DB_SYNC_WRITE(mem_handle) bus_dmamap_sync( \
418 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREWRITE)
419 #define ENA_DB_SYNC_PREREAD(mem_handle) bus_dmamap_sync( \
420 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_PREREAD)
421 #define ENA_DB_SYNC_POSTREAD(mem_handle) bus_dmamap_sync( \
422 (mem_handle)->tag, (mem_handle)->map, BUS_DMASYNC_POSTREAD)
423 #define ENA_DB_SYNC(mem_handle) ENA_DB_SYNC_WRITE(mem_handle)
424
425 #define time_after(a,b) ((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
426
427 #define VLAN_HLEN sizeof(struct ether_vlan_header)
428
429 #define prefetch(x) (void)(x)
430 #define prefetchw(x) (void)(x)
431
432 /* DMA buffers access */
433 #define dma_unmap_addr(p, name) ((p)->dma->name)
434 #define dma_unmap_addr_set(p, name, v) (((p)->dma->name) = (v))
435 #define dma_unmap_len(p, name) ((p)->name)
436 #define dma_unmap_len_set(p, name, v) (((p)->name) = (v))
437
438 #define memcpy_toio memcpy
439
440 #define ATOMIC32_INC(I32_PTR) atomic_add_int(I32_PTR, 1)
441 #define ATOMIC32_DEC(I32_PTR) atomic_add_int(I32_PTR, -1)
442 #define ATOMIC32_READ(I32_PTR) atomic_load_acq_int(I32_PTR)
443 #define ATOMIC32_SET(I32_PTR, VAL) atomic_store_rel_int(I32_PTR, VAL)
444
445 #define barrier() __asm__ __volatile__("": : :"memory")
446 #define dma_rmb() barrier()
447 #define mmiowb() barrier()
448
449 #define ACCESS_ONCE(x) (*(volatile __typeof(x) *)&(x))
450 #define READ_ONCE(x) ({ \
451 __typeof(x) __var; \
452 barrier(); \
453 __var = ACCESS_ONCE(x); \
454 barrier(); \
455 __var; \
456 })
457 #define READ_ONCE8(x) READ_ONCE(x)
458 #define READ_ONCE16(x) READ_ONCE(x)
459 #define READ_ONCE32(x) READ_ONCE(x)
460
461 #define upper_32_bits(n) ((uint32_t)(((n) >> 16) >> 16))
462 #define lower_32_bits(n) ((uint32_t)(n))
463
464 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
465
466 #define ENA_FFS(x) ffs(x)
467
468 void ena_rss_key_fill(void *key, size_t size);
469
470 #define ENA_RSS_FILL_KEY(key, size) ena_rss_key_fill(key, size)
471
472 #include "ena_defs/ena_includes.h"
473
474 #define ENA_BITS_PER_U64(bitmap) (bitcount64(bitmap))
475
476 #endif /* ENA_PLAT_H_ */
477