1 /* $NetBSD: usb_mem.c,v 1.84 2021/12/21 09:51:22 skrll Exp $ */
2
3 /*
4 * Copyright (c) 1998 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Lennart Augustsson (lennart@augustsson.net) at
9 * Carlstedt Research & Technology.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * USB DMA memory allocation.
35 * We need to allocate a lot of small (many 8 byte, some larger)
36 * memory blocks that can be used for DMA. Using the bus_dma
37 * routines directly would incur large overheads in space and time.
38 */
39
40 #include <sys/cdefs.h>
41 __KERNEL_RCSID(0, "$NetBSD: usb_mem.c,v 1.84 2021/12/21 09:51:22 skrll Exp $");
42
43 #ifdef _KERNEL_OPT
44 #include "opt_usb.h"
45 #endif
46
47 #include <sys/param.h>
48 #include <sys/bus.h>
49 #include <sys/cpu.h>
50 #include <sys/device.h> /* for usbdivar.h */
51 #include <sys/kernel.h>
52 #include <sys/kmem.h>
53 #include <sys/once.h>
54 #include <sys/queue.h>
55 #include <sys/systm.h>
56
57 #include <dev/usb/usb.h>
58 #include <dev/usb/usbdi.h>
59 #include <dev/usb/usbdivar.h> /* just for usb_dma_t */
60 #include <dev/usb/usbhist.h>
61 #include <dev/usb/usb_mem.h>
62
63 #define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(usbdebug,FMT,A,B,C,D)
64 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(usbdebug,N,FMT,A,B,C,D)
65
66 #define USB_MEM_SMALL roundup(64, CACHE_LINE_SIZE)
67 #define USB_MEM_CHUNKS 64
68 #define USB_MEM_BLOCK (USB_MEM_SMALL * USB_MEM_CHUNKS)
69
70 /* This struct is overlayed on free fragments. */
71 struct usb_frag_dma {
72 usb_dma_block_t *ufd_block;
73 u_int ufd_offs;
74 LIST_ENTRY(usb_frag_dma) ufd_next;
75 };
76
77 Static int usb_block_allocmem(bus_dma_tag_t, size_t, size_t,
78 u_int, usb_dma_block_t **);
79 Static void usb_block_freemem(usb_dma_block_t *);
80
81 LIST_HEAD(usb_dma_block_qh, usb_dma_block);
82 Static struct usb_dma_block_qh usb_blk_freelist =
83 LIST_HEAD_INITIALIZER(usb_blk_freelist);
84 kmutex_t usb_blk_lock;
85
86 #ifdef DEBUG
87 Static struct usb_dma_block_qh usb_blk_fraglist =
88 LIST_HEAD_INITIALIZER(usb_blk_fraglist);
89 Static struct usb_dma_block_qh usb_blk_fulllist =
90 LIST_HEAD_INITIALIZER(usb_blk_fulllist);
91 #endif
92 Static u_int usb_blk_nfree = 0;
93 /* XXX should have different free list for different tags (for speed) */
94 Static LIST_HEAD(, usb_frag_dma) usb_frag_freelist =
95 LIST_HEAD_INITIALIZER(usb_frag_freelist);
96
97 Static int usb_mem_init(void);
98
99 Static int
usb_mem_init(void)100 usb_mem_init(void)
101 {
102
103 mutex_init(&usb_blk_lock, MUTEX_DEFAULT, IPL_NONE);
104 return 0;
105 }
106
107 Static int
usb_block_allocmem(bus_dma_tag_t tag,size_t size,size_t align,u_int flags,usb_dma_block_t ** dmap)108 usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align,
109 u_int flags, usb_dma_block_t **dmap)
110 {
111 usb_dma_block_t *b;
112 int error;
113
114 USBHIST_FUNC();
115 USBHIST_CALLARGS(usbdebug, "size=%ju align=%ju flags=%#jx", size, align, flags, 0);
116
117 ASSERT_SLEEPABLE();
118 KASSERT(size != 0);
119 KASSERT(mutex_owned(&usb_blk_lock));
120
121 #ifdef USB_FRAG_DMA_WORKAROUND
122 flags |= USBMALLOC_ZERO;
123 #endif
124
125 bool multiseg = (flags & USBMALLOC_MULTISEG) != 0;
126 bool coherent = (flags & USBMALLOC_COHERENT) != 0;
127 bool zero = (flags & USBMALLOC_ZERO) != 0;
128 u_int dmaflags = coherent ? USB_DMA_COHERENT : 0;
129
130 /* First check the free list. */
131 LIST_FOREACH(b, &usb_blk_freelist, next) {
132 /* Don't allocate multiple segments to unwilling callers */
133 if (b->nsegs != 1 && !multiseg)
134 continue;
135 if (b->tag == tag &&
136 b->size >= size &&
137 b->align >= align &&
138 (b->flags & USB_DMA_COHERENT) == dmaflags) {
139 LIST_REMOVE(b, next);
140 usb_blk_nfree--;
141 *dmap = b;
142 if (zero) {
143 memset(b->kaddr, 0, b->size);
144 bus_dmamap_sync(b->tag, b->map, 0, b->size,
145 BUS_DMASYNC_PREWRITE);
146 }
147 DPRINTFN(6, "free list size=%ju", b->size, 0, 0, 0);
148 return 0;
149 }
150 }
151
152 DPRINTFN(6, "no freelist entry", 0, 0, 0, 0);
153 mutex_exit(&usb_blk_lock);
154
155 b = kmem_zalloc(sizeof(*b), KM_SLEEP);
156 b->tag = tag;
157 b->size = size;
158 b->align = align;
159 b->flags = dmaflags;
160
161 if (!multiseg)
162 /* Caller wants one segment */
163 b->nsegs = 1;
164 else
165 b->nsegs = howmany(size, PAGE_SIZE);
166
167 b->segs = kmem_alloc(b->nsegs * sizeof(*b->segs), KM_SLEEP);
168 b->nsegs_alloc = b->nsegs;
169
170 error = bus_dmamem_alloc(tag, b->size, align, 0, b->segs, b->nsegs,
171 &b->nsegs, BUS_DMA_WAITOK);
172 if (error)
173 goto free0;
174
175 error = bus_dmamem_map(tag, b->segs, b->nsegs, b->size, &b->kaddr,
176 BUS_DMA_WAITOK | (coherent ? BUS_DMA_COHERENT : 0));
177 if (error)
178 goto free1;
179
180 error = bus_dmamap_create(tag, b->size, b->nsegs, b->size, 0,
181 BUS_DMA_WAITOK, &b->map);
182 if (error)
183 goto unmap;
184
185 error = bus_dmamap_load(tag, b->map, b->kaddr, b->size, NULL,
186 BUS_DMA_WAITOK);
187 if (error)
188 goto destroy;
189
190 *dmap = b;
191
192 if (zero) {
193 memset(b->kaddr, 0, b->size);
194 bus_dmamap_sync(b->tag, b->map, 0, b->size,
195 BUS_DMASYNC_PREWRITE);
196 }
197
198 mutex_enter(&usb_blk_lock);
199
200 return 0;
201
202 destroy:
203 bus_dmamap_destroy(tag, b->map);
204 unmap:
205 bus_dmamem_unmap(tag, b->kaddr, b->size);
206 free1:
207 bus_dmamem_free(tag, b->segs, b->nsegs);
208 free0:
209 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs));
210 kmem_free(b, sizeof(*b));
211 mutex_enter(&usb_blk_lock);
212
213 return error;
214 }
215
216 #if 0
217 void
218 usb_block_real_freemem(usb_dma_block_t *b)
219 {
220 ASSERT_SLEEPABLE();
221
222 bus_dmamap_unload(b->tag, b->map);
223 bus_dmamap_destroy(b->tag, b->map);
224 bus_dmamem_unmap(b->tag, b->kaddr, b->size);
225 bus_dmamem_free(b->tag, b->segs, b->nsegs);
226 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs));
227 kmem_free(b, sizeof(*b));
228 }
229 #endif
230
231 #ifdef DEBUG
232 static bool
usb_valid_block_p(usb_dma_block_t * b,struct usb_dma_block_qh * qh)233 usb_valid_block_p(usb_dma_block_t *b, struct usb_dma_block_qh *qh)
234 {
235 usb_dma_block_t *xb;
236 LIST_FOREACH(xb, qh, next) {
237 if (xb == b)
238 return true;
239 }
240 return false;
241 }
242 #endif
243
244 /*
245 * Do not free the memory unconditionally since we might be called
246 * from an interrupt context and that is BAD.
247 * XXX when should we really free?
248 */
249 Static void
usb_block_freemem(usb_dma_block_t * b)250 usb_block_freemem(usb_dma_block_t *b)
251 {
252 USBHIST_FUNC();
253 USBHIST_CALLARGS(usbdebug, "size=%ju", b->size, 0, 0, 0);
254
255 KASSERT(mutex_owned(&usb_blk_lock));
256
257 #ifdef DEBUG
258 LIST_REMOVE(b, next);
259 #endif
260 LIST_INSERT_HEAD(&usb_blk_freelist, b, next);
261 usb_blk_nfree++;
262 }
263
264 int
usb_allocmem(bus_dma_tag_t tag,size_t size,size_t align,u_int flags,usb_dma_t * p)265 usb_allocmem(bus_dma_tag_t tag, size_t size, size_t align, u_int flags,
266 usb_dma_t *p)
267 {
268 usbd_status err;
269 struct usb_frag_dma *f;
270 usb_dma_block_t *b;
271 int i;
272 static ONCE_DECL(init_control);
273
274 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
275
276 ASSERT_SLEEPABLE();
277
278 RUN_ONCE(&init_control, usb_mem_init);
279
280 u_int dmaflags = (flags & USBMALLOC_COHERENT) ? USB_DMA_COHERENT : 0;
281
282 /* If the request is large then just use a full block. */
283 if (size > USB_MEM_SMALL || align > USB_MEM_SMALL) {
284 DPRINTFN(1, "large alloc %jd", size, 0, 0, 0);
285 size = (size + USB_MEM_BLOCK - 1) & ~(USB_MEM_BLOCK - 1);
286 mutex_enter(&usb_blk_lock);
287 err = usb_block_allocmem(tag, size, align, flags,
288 &p->udma_block);
289 if (!err) {
290 #ifdef DEBUG
291 LIST_INSERT_HEAD(&usb_blk_fulllist, p->udma_block, next);
292 #endif
293 p->udma_block->flags = USB_DMA_FULLBLOCK | dmaflags;
294 p->udma_offs = 0;
295 }
296 mutex_exit(&usb_blk_lock);
297 return err;
298 }
299
300 mutex_enter(&usb_blk_lock);
301 /* Check for free fragments. */
302 LIST_FOREACH(f, &usb_frag_freelist, ufd_next) {
303 KDASSERTMSG(usb_valid_block_p(f->ufd_block, &usb_blk_fraglist),
304 "%s: usb frag %p: unknown block pointer %p",
305 __func__, f, f->ufd_block);
306 if (f->ufd_block->tag == tag &&
307 (f->ufd_block->flags & USB_DMA_COHERENT) == dmaflags)
308 break;
309 }
310 if (f == NULL) {
311 DPRINTFN(1, "adding fragments", 0, 0, 0, 0);
312
313 err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL,
314 flags, &b);
315 if (err) {
316 mutex_exit(&usb_blk_lock);
317 return err;
318 }
319 #ifdef DEBUG
320 LIST_INSERT_HEAD(&usb_blk_fraglist, b, next);
321 #endif
322 b->flags = 0;
323 for (i = 0; i < USB_MEM_BLOCK; i += USB_MEM_SMALL) {
324 f = (struct usb_frag_dma *)((char *)b->kaddr + i);
325 f->ufd_block = b;
326 f->ufd_offs = i;
327 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next);
328 #ifdef USB_FRAG_DMA_WORKAROUND
329 i += 1 * USB_MEM_SMALL;
330 #endif
331 }
332 f = LIST_FIRST(&usb_frag_freelist);
333 }
334 p->udma_block = f->ufd_block;
335 p->udma_offs = f->ufd_offs;
336 #ifdef USB_FRAG_DMA_WORKAROUND
337 p->udma_offs += USB_MEM_SMALL;
338 #endif
339 LIST_REMOVE(f, ufd_next);
340 mutex_exit(&usb_blk_lock);
341 DPRINTFN(5, "use frag=%#jx size=%jd", (uintptr_t)f, size, 0, 0);
342
343 return 0;
344 }
345
346 void
usb_freemem(usb_dma_t * p)347 usb_freemem(usb_dma_t *p)
348 {
349 struct usb_frag_dma *f;
350
351 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
352
353 mutex_enter(&usb_blk_lock);
354 if (p->udma_block->flags & USB_DMA_FULLBLOCK) {
355 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fulllist),
356 "%s: dma %p: invalid block pointer %p",
357 __func__, p, p->udma_block);
358 DPRINTFN(1, "large free", 0, 0, 0, 0);
359 usb_block_freemem(p->udma_block);
360 mutex_exit(&usb_blk_lock);
361 return;
362 }
363 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fraglist),
364 "%s: dma %p: invalid block pointer %p",
365 __func__, p, p->udma_block);
366 //usb_syncmem(p, 0, USB_MEM_SMALL, BUS_DMASYNC_POSTREAD);
367 f = KERNADDR(p, 0);
368 #ifdef USB_FRAG_DMA_WORKAROUND
369 f = (void *)((uintptr_t)f - USB_MEM_SMALL);
370 #endif
371 f->ufd_block = p->udma_block;
372 f->ufd_offs = p->udma_offs;
373 #ifdef USB_FRAG_DMA_WORKAROUND
374 f->ufd_offs -= USB_MEM_SMALL;
375 #endif
376 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next);
377 mutex_exit(&usb_blk_lock);
378 DPRINTFN(5, "frag=%#jx", (uintptr_t)f, 0, 0, 0);
379 }
380
381 bus_addr_t
usb_dmaaddr(usb_dma_t * dma,unsigned int offset)382 usb_dmaaddr(usb_dma_t *dma, unsigned int offset)
383 {
384 unsigned int i;
385 bus_size_t seg_offs;
386
387 offset += dma->udma_offs;
388
389 KASSERTMSG(offset < dma->udma_block->size, "offset %d vs %zu", offset,
390 dma->udma_block->size);
391
392 if (dma->udma_block->nsegs == 1) {
393 KASSERT(dma->udma_block->map->dm_segs[0].ds_len > offset);
394 return dma->udma_block->map->dm_segs[0].ds_addr + offset;
395 }
396
397 /*
398 * Search for a bus_segment_t corresponding to this offset. With no
399 * record of the offset in the map to a particular dma_segment_t, we
400 * have to iterate from the start of the list each time. Could be
401 * improved
402 */
403 seg_offs = 0;
404 for (i = 0; i < dma->udma_block->nsegs; i++) {
405 if (seg_offs + dma->udma_block->map->dm_segs[i].ds_len > offset)
406 break;
407
408 seg_offs += dma->udma_block->map->dm_segs[i].ds_len;
409 }
410
411 KASSERT(i != dma->udma_block->nsegs);
412 offset -= seg_offs;
413 return dma->udma_block->map->dm_segs[i].ds_addr + offset;
414 }
415
416 void
usb_syncmem(usb_dma_t * p,bus_addr_t offset,bus_size_t len,int ops)417 usb_syncmem(usb_dma_t *p, bus_addr_t offset, bus_size_t len, int ops)
418 {
419
420 bus_dmamap_sync(p->udma_block->tag, p->udma_block->map,
421 p->udma_offs + offset, len, ops);
422 }
423