1 /* $NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $ */
2
3 /*
4 * Copyright (c) 1998 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Lennart Augustsson (lennart@augustsson.net) at
9 * Carlstedt Research & Technology.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 /*
34 * USB DMA memory allocation.
35 * We need to allocate a lot of small (many 8 byte, some larger)
36 * memory blocks that can be used for DMA. Using the bus_dma
37 * routines directly would incur large overheads in space and time.
38 */
39
40 #include <sys/cdefs.h>
41 __KERNEL_RCSID(0, "$NetBSD: usb_mem.c,v 1.68 2016/04/30 14:31:39 skrll Exp $");
42
43 #ifdef _KERNEL_OPT
44 #include "opt_usb.h"
45 #endif
46
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/kernel.h>
50 #include <sys/kmem.h>
51 #include <sys/queue.h>
52 #include <sys/device.h> /* for usbdivar.h */
53 #include <sys/bus.h>
54 #include <sys/cpu.h>
55 #include <sys/once.h>
56
57 #ifdef DIAGNOSTIC
58 #include <sys/proc.h>
59 #endif
60
61 #include <dev/usb/usb.h>
62 #include <dev/usb/usbdi.h>
63 #include <dev/usb/usbdivar.h> /* just for usb_dma_t */
64 #include <dev/usb/usb_mem.h>
65 #include <dev/usb/usbhist.h>
66
67 #define DPRINTF(FMT,A,B,C,D) USBHIST_LOG(usbdebug,FMT,A,B,C,D)
68 #define DPRINTFN(N,FMT,A,B,C,D) USBHIST_LOGN(usbdebug,N,FMT,A,B,C,D)
69
70 #define USB_MEM_SMALL roundup(64, CACHE_LINE_SIZE)
71 #define USB_MEM_CHUNKS 64
72 #define USB_MEM_BLOCK (USB_MEM_SMALL * USB_MEM_CHUNKS)
73
74 /* This struct is overlayed on free fragments. */
75 struct usb_frag_dma {
76 usb_dma_block_t *ufd_block;
77 u_int ufd_offs;
78 LIST_ENTRY(usb_frag_dma) ufd_next;
79 };
80
81 Static usbd_status usb_block_allocmem(bus_dma_tag_t, size_t, size_t,
82 usb_dma_block_t **, bool);
83 Static void usb_block_freemem(usb_dma_block_t *);
84
85 LIST_HEAD(usb_dma_block_qh, usb_dma_block);
86 Static struct usb_dma_block_qh usb_blk_freelist =
87 LIST_HEAD_INITIALIZER(usb_blk_freelist);
88 kmutex_t usb_blk_lock;
89
90 #ifdef DEBUG
91 Static struct usb_dma_block_qh usb_blk_fraglist =
92 LIST_HEAD_INITIALIZER(usb_blk_fraglist);
93 Static struct usb_dma_block_qh usb_blk_fulllist =
94 LIST_HEAD_INITIALIZER(usb_blk_fulllist);
95 #endif
96 Static u_int usb_blk_nfree = 0;
97 /* XXX should have different free list for different tags (for speed) */
98 Static LIST_HEAD(, usb_frag_dma) usb_frag_freelist =
99 LIST_HEAD_INITIALIZER(usb_frag_freelist);
100
101 Static int usb_mem_init(void);
102
103 Static int
usb_mem_init(void)104 usb_mem_init(void)
105 {
106
107 mutex_init(&usb_blk_lock, MUTEX_DEFAULT, IPL_NONE);
108 return 0;
109 }
110
111 Static usbd_status
usb_block_allocmem(bus_dma_tag_t tag,size_t size,size_t align,usb_dma_block_t ** dmap,bool multiseg)112 usb_block_allocmem(bus_dma_tag_t tag, size_t size, size_t align,
113 usb_dma_block_t **dmap, bool multiseg)
114 {
115 usb_dma_block_t *b;
116 int error;
117
118 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
119 DPRINTFN(5, "size=%zu align=%zu", size, align, 0, 0);
120
121 ASSERT_SLEEPABLE();
122 KASSERT(size != 0);
123 KASSERT(mutex_owned(&usb_blk_lock));
124
125 /* First check the free list. */
126 LIST_FOREACH(b, &usb_blk_freelist, next) {
127 /* Don't allocate multiple segments to unwilling callers */
128 if (b->nsegs != 1 && !multiseg)
129 continue;
130 if (b->tag == tag && b->size >= size && b->align >= align) {
131 LIST_REMOVE(b, next);
132 usb_blk_nfree--;
133 *dmap = b;
134 DPRINTFN(6, "free list size=%zu", b->size, 0, 0, 0);
135 return USBD_NORMAL_COMPLETION;
136 }
137 }
138
139 DPRINTFN(6, "no free", 0, 0, 0, 0);
140 mutex_exit(&usb_blk_lock);
141
142 b = kmem_zalloc(sizeof(*b), KM_SLEEP);
143 if (b == NULL) {
144 goto fail;
145 }
146
147 b->tag = tag;
148 b->size = size;
149 b->align = align;
150
151 if (!multiseg)
152 /* Caller wants one segment */
153 b->nsegs = 1;
154 else
155 b->nsegs = (size + (PAGE_SIZE-1)) / PAGE_SIZE;
156
157 b->segs = kmem_alloc(b->nsegs * sizeof(*b->segs), KM_SLEEP);
158 if (b->segs == NULL) {
159 kmem_free(b, sizeof(*b));
160 goto fail;
161 }
162 b->nsegs_alloc = b->nsegs;
163
164 error = bus_dmamem_alloc(tag, b->size, align, 0,
165 b->segs, b->nsegs,
166 &b->nsegs, BUS_DMA_WAITOK);
167 if (error)
168 goto free0;
169
170 error = bus_dmamem_map(tag, b->segs, b->nsegs, b->size,
171 &b->kaddr, BUS_DMA_WAITOK|BUS_DMA_COHERENT);
172 if (error)
173 goto free1;
174
175 error = bus_dmamap_create(tag, b->size, b->nsegs, b->size,
176 0, BUS_DMA_WAITOK, &b->map);
177 if (error)
178 goto unmap;
179
180 error = bus_dmamap_load(tag, b->map, b->kaddr, b->size, NULL,
181 BUS_DMA_WAITOK);
182 if (error)
183 goto destroy;
184
185 *dmap = b;
186 #ifdef USB_FRAG_DMA_WORKAROUND
187 memset(b->kaddr, 0, b->size);
188 #endif
189 mutex_enter(&usb_blk_lock);
190
191 return USBD_NORMAL_COMPLETION;
192
193 destroy:
194 bus_dmamap_destroy(tag, b->map);
195 unmap:
196 bus_dmamem_unmap(tag, b->kaddr, b->size);
197 free1:
198 bus_dmamem_free(tag, b->segs, b->nsegs);
199 free0:
200 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs));
201 kmem_free(b, sizeof(*b));
202 fail:
203 mutex_enter(&usb_blk_lock);
204
205 return USBD_NOMEM;
206 }
207
208 #if 0
209 void
210 usb_block_real_freemem(usb_dma_block_t *b)
211 {
212 #ifdef DIAGNOSTIC
213 if (cpu_softintr_p() || cpu_intr_p()) {
214 printf("usb_block_real_freemem: in interrupt context\n");
215 return;
216 }
217 #endif
218 bus_dmamap_unload(b->tag, b->map);
219 bus_dmamap_destroy(b->tag, b->map);
220 bus_dmamem_unmap(b->tag, b->kaddr, b->size);
221 bus_dmamem_free(b->tag, b->segs, b->nsegs);
222 kmem_free(b->segs, b->nsegs_alloc * sizeof(*b->segs));
223 kmem_free(b, sizeof(*b));
224 }
225 #endif
226
227 #ifdef DEBUG
228 static bool
usb_valid_block_p(usb_dma_block_t * b,struct usb_dma_block_qh * qh)229 usb_valid_block_p(usb_dma_block_t *b, struct usb_dma_block_qh *qh)
230 {
231 usb_dma_block_t *xb;
232 LIST_FOREACH(xb, qh, next) {
233 if (xb == b)
234 return true;
235 }
236 return false;
237 }
238 #endif
239
240 /*
241 * Do not free the memory unconditionally since we might be called
242 * from an interrupt context and that is BAD.
243 * XXX when should we really free?
244 */
245 Static void
usb_block_freemem(usb_dma_block_t * b)246 usb_block_freemem(usb_dma_block_t *b)
247 {
248
249 KASSERT(mutex_owned(&usb_blk_lock));
250
251 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
252 DPRINTFN(6, "size=%zu", b->size, 0, 0, 0);
253 #ifdef DEBUG
254 LIST_REMOVE(b, next);
255 #endif
256 LIST_INSERT_HEAD(&usb_blk_freelist, b, next);
257 usb_blk_nfree++;
258 }
259
260 usbd_status
usb_allocmem(struct usbd_bus * bus,size_t size,size_t align,usb_dma_t * p)261 usb_allocmem(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p)
262 {
263
264 return usb_allocmem_flags(bus, size, align, p, 0);
265 }
266
267 usbd_status
usb_allocmem_flags(struct usbd_bus * bus,size_t size,size_t align,usb_dma_t * p,int flags)268 usb_allocmem_flags(struct usbd_bus *bus, size_t size, size_t align, usb_dma_t *p,
269 int flags)
270 {
271 bus_dma_tag_t tag = bus->ub_dmatag;
272 usbd_status err;
273 struct usb_frag_dma *f;
274 usb_dma_block_t *b;
275 int i;
276 static ONCE_DECL(init_control);
277 bool frag;
278
279 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
280
281 ASSERT_SLEEPABLE();
282
283 RUN_ONCE(&init_control, usb_mem_init);
284
285 frag = (flags & USBMALLOC_MULTISEG);
286
287 /* If the request is large then just use a full block. */
288 if (size > USB_MEM_SMALL || align > USB_MEM_SMALL) {
289 DPRINTFN(1, "large alloc %d", size, 0, 0, 0);
290 size = (size + USB_MEM_BLOCK - 1) & ~(USB_MEM_BLOCK - 1);
291 mutex_enter(&usb_blk_lock);
292 err = usb_block_allocmem(tag, size, align, &p->udma_block, frag);
293 if (!err) {
294 #ifdef DEBUG
295 LIST_INSERT_HEAD(&usb_blk_fulllist, p->udma_block, next);
296 #endif
297 p->udma_block->flags = USB_DMA_FULLBLOCK;
298 p->udma_offs = 0;
299 }
300 mutex_exit(&usb_blk_lock);
301 return err;
302 }
303
304 mutex_enter(&usb_blk_lock);
305 /* Check for free fragments. */
306 LIST_FOREACH(f, &usb_frag_freelist, ufd_next) {
307 KDASSERTMSG(usb_valid_block_p(f->ufd_block, &usb_blk_fraglist),
308 "%s: usb frag %p: unknown block pointer %p",
309 __func__, f, f->ufd_block);
310 if (f->ufd_block->tag == tag)
311 break;
312 }
313 if (f == NULL) {
314 DPRINTFN(1, "adding fragments", 0, 0, 0, 0);
315 err = usb_block_allocmem(tag, USB_MEM_BLOCK, USB_MEM_SMALL, &b,
316 false);
317 if (err) {
318 mutex_exit(&usb_blk_lock);
319 return err;
320 }
321 #ifdef DEBUG
322 LIST_INSERT_HEAD(&usb_blk_fraglist, b, next);
323 #endif
324 b->flags = 0;
325 for (i = 0; i < USB_MEM_BLOCK; i += USB_MEM_SMALL) {
326 f = (struct usb_frag_dma *)((char *)b->kaddr + i);
327 f->ufd_block = b;
328 f->ufd_offs = i;
329 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next);
330 #ifdef USB_FRAG_DMA_WORKAROUND
331 i += 1 * USB_MEM_SMALL;
332 #endif
333 }
334 f = LIST_FIRST(&usb_frag_freelist);
335 }
336 p->udma_block = f->ufd_block;
337 p->udma_offs = f->ufd_offs;
338 #ifdef USB_FRAG_DMA_WORKAROUND
339 p->udma_offs += USB_MEM_SMALL;
340 #endif
341 LIST_REMOVE(f, ufd_next);
342 mutex_exit(&usb_blk_lock);
343 DPRINTFN(5, "use frag=%p size=%d", f, size, 0, 0);
344
345 return USBD_NORMAL_COMPLETION;
346 }
347
348 void
usb_freemem(struct usbd_bus * bus,usb_dma_t * p)349 usb_freemem(struct usbd_bus *bus, usb_dma_t *p)
350 {
351 struct usb_frag_dma *f;
352
353 USBHIST_FUNC(); USBHIST_CALLED(usbdebug);
354
355 mutex_enter(&usb_blk_lock);
356 if (p->udma_block->flags & USB_DMA_FULLBLOCK) {
357 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fulllist),
358 "%s: dma %p: invalid block pointer %p",
359 __func__, p, p->udma_block);
360 DPRINTFN(1, "large free", 0, 0, 0, 0);
361 usb_block_freemem(p->udma_block);
362 mutex_exit(&usb_blk_lock);
363 return;
364 }
365 KDASSERTMSG(usb_valid_block_p(p->udma_block, &usb_blk_fraglist),
366 "%s: dma %p: invalid block pointer %p",
367 __func__, p, p->udma_block);
368 //usb_syncmem(p, 0, USB_MEM_SMALL, BUS_DMASYNC_POSTREAD);
369 f = KERNADDR(p, 0);
370 #ifdef USB_FRAG_DMA_WORKAROUND
371 f = (void *)((uintptr_t)f - USB_MEM_SMALL);
372 #endif
373 f->ufd_block = p->udma_block;
374 f->ufd_offs = p->udma_offs;
375 #ifdef USB_FRAG_DMA_WORKAROUND
376 f->ufd_offs -= USB_MEM_SMALL;
377 #endif
378 LIST_INSERT_HEAD(&usb_frag_freelist, f, ufd_next);
379 mutex_exit(&usb_blk_lock);
380 DPRINTFN(5, "frag=%p", f, 0, 0, 0);
381 }
382
383 bus_addr_t
usb_dmaaddr(usb_dma_t * dma,unsigned int offset)384 usb_dmaaddr(usb_dma_t *dma, unsigned int offset)
385 {
386 unsigned int i;
387 bus_size_t seg_offs;
388
389 offset += dma->udma_offs;
390
391 KASSERTMSG(offset < dma->udma_block->size, "offset %d vs %zu", offset,
392 dma->udma_block->size);
393
394 if (dma->udma_block->nsegs == 1) {
395 KASSERT(dma->udma_block->map->dm_segs[0].ds_len > offset);
396 return dma->udma_block->map->dm_segs[0].ds_addr + offset;
397 }
398
399 /*
400 * Search for a bus_segment_t corresponding to this offset. With no
401 * record of the offset in the map to a particular dma_segment_t, we
402 * have to iterate from the start of the list each time. Could be
403 * improved
404 */
405 seg_offs = 0;
406 for (i = 0; i < dma->udma_block->nsegs; i++) {
407 if (seg_offs + dma->udma_block->map->dm_segs[i].ds_len > offset)
408 break;
409
410 seg_offs += dma->udma_block->map->dm_segs[i].ds_len;
411 }
412
413 KASSERT(i != dma->udma_block->nsegs);
414 offset -= seg_offs;
415 return dma->udma_block->map->dm_segs[i].ds_addr + offset;
416 }
417
418 void
usb_syncmem(usb_dma_t * p,bus_addr_t offset,bus_size_t len,int ops)419 usb_syncmem(usb_dma_t *p, bus_addr_t offset, bus_size_t len, int ops)
420 {
421
422 bus_dmamap_sync(p->udma_block->tag, p->udma_block->map, p->udma_offs + offset,
423 len, ops);
424 }
425