1 /* $NetBSD: bus_dma.c,v 1.55 2022/07/26 20:08:56 andvar Exp $ */
2
3 /*-
4 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #define _POWERPC_BUS_DMA_PRIVATE
34
35 #include <sys/cdefs.h>
36 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.55 2022/07/26 20:08:56 andvar Exp $");
37
38 #ifdef _KERNEL_OPT
39 #include "opt_ppcarch.h"
40 #endif
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/device.h>
46 #include <sys/kmem.h>
47 #include <sys/proc.h>
48 #include <sys/mbuf.h>
49 #include <sys/bus.h>
50 #include <sys/intr.h>
51
52 #include <uvm/uvm.h>
53 #include <uvm/uvm_physseg.h>
54
55 #if defined(PPC_BOOKE)
56 #define EIEIO __asm volatile("mbar\t0" ::: "memory")
57 #define SYNC __asm volatile("msync" ::: "memory")
58 #elif defined(PPC_IBM4XX) && !defined(PPC_IBM440)
59 /* eieio is implemented as sync */
60 #define EIEIO __asm volatile("eieio" ::: "memory")
61 #define SYNC /* nothing */
62 #else
63 #define EIEIO __asm volatile("eieio" ::: "memory")
64 #define SYNC __asm volatile("sync" ::: "memory")
65 #endif
66
67 int _bus_dmamap_load_buffer (bus_dma_tag_t, bus_dmamap_t, void *,
68 bus_size_t, struct vmspace *, int, paddr_t *, int *, int);
69
70 static inline void
dcbst(paddr_t pa,long len,int dcache_line_size)71 dcbst(paddr_t pa, long len, int dcache_line_size)
72 {
73 paddr_t epa;
74 for (epa = pa + len; pa < epa; pa += dcache_line_size)
75 __asm volatile("dcbst 0,%0" :: "r"(pa) : "memory");
76 }
77
78 static inline void
dcbi(paddr_t pa,long len,int dcache_line_size)79 dcbi(paddr_t pa, long len, int dcache_line_size)
80 {
81 paddr_t epa;
82 for (epa = pa + len; pa < epa; pa += dcache_line_size)
83 __asm volatile("dcbi 0,%0" :: "r"(pa) : "memory");
84 }
85
86 static inline void
dcbf(paddr_t pa,long len,int dcache_line_size)87 dcbf(paddr_t pa, long len, int dcache_line_size)
88 {
89 paddr_t epa;
90 for (epa = pa + len; pa < epa; pa += dcache_line_size)
91 __asm volatile("dcbf 0,%0" :: "r"(pa) : "memory");
92 }
93
94 /*
95 * Common function for DMA map creation. May be called by bus-specific
96 * DMA map creation functions.
97 */
98 int
_bus_dmamap_create(bus_dma_tag_t t,bus_size_t size,int nsegments,bus_size_t maxsegsz,bus_size_t boundary,int flags,bus_dmamap_t * dmamp)99 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
100 {
101 struct powerpc_bus_dmamap *map;
102 void *mapstore;
103 size_t mapsize;
104
105 /*
106 * Allocate and initialize the DMA map. The end of the map
107 * is a variable-sized array of segments, so we allocate enough
108 * room for them in one shot.
109 *
110 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation
111 * of ALLOCNOW notifies others that we've reserved these resources,
112 * and they are not to be freed.
113 *
114 * The bus_dmamap_t includes one bus_dma_segment_t, hence
115 * the (nsegments - 1).
116 */
117 mapsize = sizeof(*map) + sizeof(bus_dma_segment_t [nsegments - 1]);
118 if ((mapstore = kmem_intr_alloc(mapsize,
119 (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP)) == NULL)
120 return (ENOMEM);
121
122 memset(mapstore, 0, mapsize);
123 map = (struct powerpc_bus_dmamap *)mapstore;
124 map->_dm_size = size;
125 map->_dm_segcnt = nsegments;
126 map->_dm_maxmaxsegsz = maxsegsz;
127 map->_dm_boundary = boundary;
128 map->_dm_bounce_thresh = t->_bounce_thresh;
129 map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT);
130 map->dm_maxsegsz = maxsegsz;
131 map->dm_mapsize = 0; /* no valid mappings */
132 map->dm_nsegs = 0;
133
134 *dmamp = map;
135 return (0);
136 }
137
138 /*
139 * Common function for DMA map destruction. May be called by bus-specific
140 * DMA map destruction functions.
141 */
142 void
_bus_dmamap_destroy(bus_dma_tag_t t,bus_dmamap_t map)143 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
144 {
145
146 size_t mapsize = sizeof(*map)
147 + sizeof(bus_dma_segment_t [map->_dm_segcnt - 1]);
148 kmem_intr_free(map, mapsize);
149 }
150
151 /*
152 * Utility function to load a linear buffer. lastaddrp holds state
153 * between invocations (for multiple-buffer loads). segp contains
154 * the starting segment on entrance, and the ending segment on exit.
155 * first indicates if this is the first invocation of this function.
156 */
157 int
_bus_dmamap_load_buffer(bus_dma_tag_t t,bus_dmamap_t map,void * buf,bus_size_t buflen,struct vmspace * vm,int flags,paddr_t * lastaddrp,int * segp,int first)158 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct vmspace *vm, int flags, paddr_t *lastaddrp, int *segp, int first)
159 {
160 bus_size_t sgsize;
161 bus_addr_t curaddr, lastaddr, baddr, bmask;
162 vaddr_t vaddr = (vaddr_t)buf;
163 int seg;
164
165 // printf("%s(%p,%p,%p,%u,%p,%#x,%p,%p,%u)\n", __func__,
166 // t, map, buf, buflen, vm, flags, lastaddrp, segp, first);
167
168 lastaddr = *lastaddrp;
169 bmask = ~(map->_dm_boundary - 1);
170
171 for (seg = *segp; buflen > 0 ; ) {
172 /*
173 * Get the physical address for this segment.
174 */
175 if (!VMSPACE_IS_KERNEL_P(vm))
176 (void) pmap_extract(vm_map_pmap(&vm->vm_map),
177 vaddr, (void *)&curaddr);
178 else
179 curaddr = vtophys(vaddr);
180
181 /*
182 * If we're beyond the bounce threshold, notify
183 * the caller.
184 */
185 if (map->_dm_bounce_thresh != 0 &&
186 curaddr >= map->_dm_bounce_thresh)
187 return (EINVAL);
188
189 /*
190 * Compute the segment size, and adjust counts.
191 */
192 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
193 if (buflen < sgsize)
194 sgsize = buflen;
195 sgsize = uimin(sgsize, map->dm_maxsegsz);
196
197 /*
198 * Make sure we don't cross any boundaries.
199 */
200 if (map->_dm_boundary > 0) {
201 baddr = (curaddr + map->_dm_boundary) & bmask;
202 if (sgsize > (baddr - curaddr))
203 sgsize = (baddr - curaddr);
204 }
205
206 /*
207 * Insert chunk into a segment, coalescing with
208 * the previous segment if possible.
209 */
210 if (first) {
211 map->dm_segs[seg].ds_addr = PHYS_TO_BUS_MEM(t, curaddr);
212 map->dm_segs[seg].ds_len = sgsize;
213 first = 0;
214 } else {
215 if (curaddr == lastaddr &&
216 (map->dm_segs[seg].ds_len + sgsize) <=
217 map->dm_maxsegsz &&
218 (map->_dm_boundary == 0 ||
219 (map->dm_segs[seg].ds_addr & bmask) ==
220 (PHYS_TO_BUS_MEM(t, curaddr) & bmask)))
221 map->dm_segs[seg].ds_len += sgsize;
222 else {
223 if (++seg >= map->_dm_segcnt)
224 break;
225 map->dm_segs[seg].ds_addr =
226 PHYS_TO_BUS_MEM(t, curaddr);
227 map->dm_segs[seg].ds_len = sgsize;
228 }
229 }
230
231 lastaddr = curaddr + sgsize;
232 vaddr += sgsize;
233 buflen -= sgsize;
234 }
235
236 *segp = seg;
237 *lastaddrp = lastaddr;
238
239 /*
240 * Did we fit?
241 */
242 if (buflen != 0)
243 return (EFBIG); /* XXX better return value here? */
244
245 return (0);
246 }
247
248 /*
249 * Common function for loading a DMA map with a linear buffer. May
250 * be called by bus-specific DMA map load functions.
251 */
252 int
_bus_dmamap_load(bus_dma_tag_t t,bus_dmamap_t map,void * buf,bus_size_t buflen,struct proc * p,int flags)253 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags)
254 {
255 paddr_t lastaddr = 0;
256 int seg, error;
257 struct vmspace *vm;
258
259 /*
260 * Make sure that on error condition we return "no valid mappings".
261 */
262 map->dm_mapsize = 0;
263 map->dm_nsegs = 0;
264 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
265
266 if (buflen > map->_dm_size)
267 return (EINVAL);
268
269 if (p != NULL) {
270 vm = p->p_vmspace;
271 } else {
272 vm = vmspace_kernel();
273 }
274
275 seg = 0;
276 error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags,
277 &lastaddr, &seg, 1);
278 if (error == 0) {
279 map->dm_mapsize = buflen;
280 map->dm_nsegs = seg + 1;
281 }
282 return (error);
283 }
284
285 /*
286 * Like _bus_dmamap_load(), but for mbufs.
287 */
288 int
_bus_dmamap_load_mbuf(bus_dma_tag_t t,bus_dmamap_t map,struct mbuf * m0,int flags)289 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags)
290 {
291 paddr_t lastaddr = 0;
292 int seg, error, first;
293 struct mbuf *m;
294
295 /*
296 * Make sure that on error condition we return "no valid mappings."
297 */
298 map->dm_mapsize = 0;
299 map->dm_nsegs = 0;
300 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
301
302 #ifdef DIAGNOSTIC
303 if ((m0->m_flags & M_PKTHDR) == 0)
304 panic("_bus_dmamap_load_mbuf: no packet header");
305 #endif
306
307 if (m0->m_pkthdr.len > map->_dm_size)
308 return (EINVAL);
309
310 first = 1;
311 seg = 0;
312 error = 0;
313 for (m = m0; m != NULL && error == 0; m = m->m_next, first = 0) {
314 if (m->m_len == 0)
315 continue;
316 #ifdef POOL_VTOPHYS
317 /* XXX Could be better about coalescing. */
318 /* XXX Doesn't check boundaries. */
319 switch (m->m_flags & (M_EXT|M_EXT_CLUSTER)) {
320 case M_EXT|M_EXT_CLUSTER:
321 /* XXX KDASSERT */
322 KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID);
323 lastaddr = m->m_ext.ext_paddr +
324 (m->m_data - m->m_ext.ext_buf);
325 have_addr:
326 if (first == 0 && ++seg >= map->_dm_segcnt) {
327 error = EFBIG;
328 continue;
329 }
330 map->dm_segs[seg].ds_addr =
331 PHYS_TO_BUS_MEM(t, lastaddr);
332 map->dm_segs[seg].ds_len = m->m_len;
333 lastaddr += m->m_len;
334 continue;
335
336 case 0:
337 lastaddr = m->m_paddr + M_BUFOFFSET(m) +
338 (m->m_data - M_BUFADDR(m));
339 goto have_addr;
340
341 default:
342 break;
343 }
344 #endif
345 error = _bus_dmamap_load_buffer(t, map, m->m_data,
346 m->m_len, vmspace_kernel(), flags, &lastaddr, &seg, first);
347 }
348 if (error == 0) {
349 map->dm_mapsize = m0->m_pkthdr.len;
350 map->dm_nsegs = seg + 1;
351 }
352 return (error);
353 }
354
355 /*
356 * Like _bus_dmamap_load(), but for uios.
357 */
358 int
_bus_dmamap_load_uio(bus_dma_tag_t t,bus_dmamap_t map,struct uio * uio,int flags)359 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
360 {
361 paddr_t lastaddr = 0;
362 int seg, i, error, first;
363 bus_size_t minlen, resid;
364 struct iovec *iov;
365 void *addr;
366
367 /*
368 * Make sure that on error condition we return "no valid mappings."
369 */
370 map->dm_mapsize = 0;
371 map->dm_nsegs = 0;
372 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
373
374 resid = uio->uio_resid;
375 iov = uio->uio_iov;
376
377 first = 1;
378 seg = 0;
379 error = 0;
380 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
381 /*
382 * Now at the first iovec to load. Load each iovec
383 * until we have exhausted the residual count.
384 */
385 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
386 addr = (void *)iov[i].iov_base;
387
388 error = _bus_dmamap_load_buffer(t, map, addr, minlen,
389 uio->uio_vmspace, flags, &lastaddr, &seg, first);
390 first = 0;
391
392 resid -= minlen;
393 }
394 if (error == 0) {
395 map->dm_mapsize = uio->uio_resid;
396 map->dm_nsegs = seg + 1;
397 }
398 return (error);
399 }
400
401 /*
402 * Like _bus_dmamap_load(), but for raw memory allocated with
403 * bus_dmamem_alloc().
404 *
405 * XXX This is too much copypasta of _bus_dmamap_load_buffer.
406 */
407 int
_bus_dmamap_load_raw(bus_dma_tag_t t,bus_dmamap_t map,bus_dma_segment_t * segs,int nsegs,bus_size_t size,int flags)408 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
409 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
410 {
411 bus_size_t sgsize, isgsize;
412 bus_size_t busaddr, curaddr, lastaddr, baddr, bmask;
413 int seg, iseg, first;
414
415 if (size == 0)
416 return 0;
417
418 lastaddr = 0;
419 bmask = ~(map->_dm_boundary - 1);
420
421 first = 0;
422 iseg = 0;
423 busaddr = segs[iseg].ds_addr;
424 isgsize = segs[iseg].ds_len;
425 for (seg = 0; size > 0;) {
426 /*
427 * Get the physical address for this segment.
428 */
429 curaddr = BUS_MEM_TO_PHYS(t, busaddr);
430
431 /*
432 * If we're beyond the bounce threshold, notify
433 * the caller.
434 */
435 if (map->_dm_bounce_thresh != 0 &&
436 curaddr >= map->_dm_bounce_thresh)
437 return EINVAL;
438
439 /*
440 * Compute the segment size, and adjust counts.
441 */
442 sgsize = PAGE_SIZE - ((u_long)curaddr & PGOFSET);
443 sgsize = MIN(sgsize, isgsize);
444 sgsize = MIN(sgsize, size);
445 sgsize = MIN(sgsize, map->dm_maxsegsz);
446
447 /*
448 * Make sure we don't cross any boundaries.
449 */
450 if (map->_dm_boundary > 0) {
451 baddr = (curaddr + map->_dm_boundary) & bmask;
452 if (sgsize > (baddr - curaddr))
453 sgsize = (baddr - curaddr);
454 }
455
456 /*
457 * Insert chunk into a segment, coalescing with
458 * the previous segment if possible.
459 */
460 if (first) {
461 map->dm_segs[seg].ds_addr =
462 PHYS_TO_BUS_MEM(t, curaddr);
463 map->dm_segs[seg].ds_len = sgsize;
464 first = 0;
465 } else {
466 if (curaddr == lastaddr &&
467 (map->dm_segs[seg].ds_len + sgsize) <=
468 map->dm_maxsegsz &&
469 (map->_dm_boundary == 0 ||
470 (map->dm_segs[seg].ds_addr & bmask) ==
471 (PHYS_TO_BUS_MEM(t, curaddr) & bmask)))
472 map->dm_segs[seg].ds_len += sgsize;
473 else {
474 if (++seg >= map->_dm_segcnt)
475 break;
476 map->dm_segs[seg].ds_addr =
477 PHYS_TO_BUS_MEM(t, curaddr);
478 map->dm_segs[seg].ds_len = sgsize;
479 }
480 }
481
482 lastaddr = curaddr + sgsize;
483 size -= sgsize;
484 if ((isgsize -= sgsize) == 0) {
485 iseg++;
486 KASSERT(iseg < nsegs);
487 busaddr = segs[iseg].ds_addr;
488 isgsize = segs[iseg].ds_len;
489 }
490 }
491
492 if (size > 0)
493 return EFBIG;
494
495 return 0;
496 }
497
498 /*
499 * Common function for unloading a DMA map. May be called by
500 * chipset-specific DMA map unload functions.
501 */
502 void
_bus_dmamap_unload(bus_dma_tag_t t,bus_dmamap_t map)503 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
504 {
505
506 /*
507 * No resources to free; just mark the mappings as
508 * invalid.
509 */
510 map->dm_maxsegsz = map->_dm_maxmaxsegsz;
511 map->dm_mapsize = 0;
512 map->dm_nsegs = 0;
513 }
514
515 /*
516 * Common function for DMA map synchronization. May be called
517 * by chipset-specific DMA map synchronization functions.
518 */
519 void
_bus_dmamap_sync(bus_dma_tag_t t,bus_dmamap_t map,bus_addr_t offset,bus_size_t len,int ops)520 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
521 {
522 const int dcache_line_size = curcpu()->ci_ci.dcache_line_size;
523 const bus_dma_segment_t *ds = map->dm_segs;
524
525 // printf("%s(%p,%p,%#x,%u,%#x) from %p\n", __func__,
526 // t, map, offset, len, ops, __builtin_return_address(0));
527
528 if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
529 (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
530 panic("_bus_dmamap_sync: invalid ops %#x", ops);
531
532 #ifdef DIAGNOSTIC
533 if (offset + len > map->dm_mapsize)
534 panic("%s: ops %#x mapsize %u: bad offset (%u) and/or length (%u)", __func__, ops, map->dm_mapsize, offset, len);
535 #endif
536
537 /*
538 * Skip leading amount
539 */
540 while (offset >= ds->ds_len) {
541 offset -= ds->ds_len;
542 ds++;
543 }
544 EIEIO;
545 for (; len > 0; ds++, offset = 0) {
546 bus_size_t seglen = ds->ds_len - offset;
547 bus_addr_t addr = BUS_MEM_TO_PHYS(t, ds->ds_addr) + offset;
548 if (seglen > len)
549 seglen = len;
550 len -= seglen;
551 KASSERT(ds < &map->dm_segs[map->dm_nsegs]);
552 /*
553 * Readjust things to start on cacheline boundarys
554 */
555 offset = (addr & (dcache_line_size-1));
556 seglen += offset;
557 addr -= offset;
558 /*
559 * Now do the appropriate thing.
560 */
561 switch (ops) {
562 case BUS_DMASYNC_PREWRITE:
563 /*
564 * Make sure cache contents are in memory for the DMA.
565 */
566 dcbst(addr, seglen, dcache_line_size);
567 break;
568 case BUS_DMASYNC_PREREAD:
569 /*
570 * If the region to be invalidated doesn't fall on
571 * cacheline boundary, flush that cacheline so we
572 * preserve the leading content.
573 */
574 if (offset) {
575 dcbf(addr, 1, 1);
576 /*
577 * If we are doing <= one cache line, stop now.
578 */
579 if (seglen <= dcache_line_size)
580 break;
581 /*
582 * Advance one cache line since we've flushed
583 * this one.
584 */
585 addr += dcache_line_size;
586 seglen -= dcache_line_size;
587 }
588 /*
589 * If the byte after the region to be invalidated
590 * doesn't fall on cacheline boundary, flush that
591 * cacheline so we preserve the trailing content.
592 */
593 if (seglen & (dcache_line_size-1)) {
594 dcbf(addr + seglen, 1, 1);
595 if (seglen <= dcache_line_size)
596 break;
597 /*
598 * Truncate the length to a multiple of a
599 * dcache line size. No reason to flush
600 * the last entry again.
601 */
602 seglen &= ~(dcache_line_size - 1);
603 }
604 SYNC; /* is this needed? */
605 EIEIO; /* is this needed? */
606 /* FALLTHROUGH */
607 case BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE:
608 case BUS_DMASYNC_POSTREAD:
609 /*
610 * The contents will have changed, make sure to remove
611 * them from the cache. Note: some implementation
612 * implement dcbi identically to dcbf. Thus if the
613 * cacheline has data, it will be written to memory.
614 * If the DMA is updating the same cacheline at the
615 * time, bad things can happen.
616 */
617 dcbi(addr, seglen, dcache_line_size);
618 break;
619 case BUS_DMASYNC_POSTWRITE:
620 /*
621 * Do nothing.
622 */
623 break;
624 case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE:
625 /*
626 * Force it to memory and remove from cache.
627 */
628 dcbf(addr, seglen, dcache_line_size);
629 break;
630 }
631 }
632 __asm volatile("sync");
633 }
634
635 /*
636 * Common function for DMA-safe memory allocation. May be called
637 * by bus-specific DMA memory allocation functions.
638 */
639 int
_bus_dmamem_alloc(bus_dma_tag_t t,bus_size_t size,bus_size_t alignment,bus_size_t boundary,bus_dma_segment_t * segs,int nsegs,int * rsegs,int flags)640 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags)
641 {
642 paddr_t start = 0xffffffff, end = 0;
643 uvm_physseg_t bank;
644
645 for (bank = uvm_physseg_get_first();
646 uvm_physseg_valid_p(bank);
647 bank = uvm_physseg_get_next(bank)) {
648 if (start > ptoa(uvm_physseg_get_avail_start(bank)))
649 start = ptoa(uvm_physseg_get_avail_start(bank));
650 if (end < ptoa(uvm_physseg_get_avail_end(bank)))
651 end = ptoa(uvm_physseg_get_avail_end(bank));
652 }
653
654 return _bus_dmamem_alloc_range(t, size, alignment, boundary, segs,
655 nsegs, rsegs, flags, start, end - PAGE_SIZE);
656 }
657
658 /*
659 * Common function for freeing DMA-safe memory. May be called by
660 * bus-specific DMA memory free functions.
661 */
662 void
_bus_dmamem_free(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs)663 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
664 {
665 struct vm_page *m;
666 bus_addr_t addr;
667 struct pglist mlist;
668 int curseg;
669
670 /*
671 * Build a list of pages to free back to the VM system.
672 */
673 TAILQ_INIT(&mlist);
674 for (curseg = 0; curseg < nsegs; curseg++) {
675 for (addr = BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr);
676 addr < (BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr)
677 + segs[curseg].ds_len);
678 addr += PAGE_SIZE) {
679 m = PHYS_TO_VM_PAGE(addr);
680 TAILQ_INSERT_TAIL(&mlist, m, pageq.queue);
681 }
682 }
683
684 uvm_pglistfree(&mlist);
685 }
686
687 /*
688 * Common function for mapping DMA-safe memory. May be called by
689 * bus-specific DMA memory map functions.
690 */
691 int
_bus_dmamem_map(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,size_t size,void ** kvap,int flags)692 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size, void **kvap, int flags)
693 {
694 vaddr_t va;
695 bus_addr_t addr;
696 int curseg;
697 const uvm_flag_t kmflags =
698 (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0;
699
700 size = round_page(size);
701
702 #ifdef PMAP_MAP_POOLPAGE
703 /*
704 * If we are mapping a cacheable physically contiguous segment, treat
705 * it as if we are mapping a poolpage and avoid consuming any KVAs.
706 */
707 if (nsegs == 1 && (flags & BUS_DMA_DONTCACHE) == 0) {
708 KASSERT(size == segs->ds_len);
709 addr = BUS_MEM_TO_PHYS(t, segs->ds_addr);
710 *kvap = (void *)PMAP_MAP_POOLPAGE(addr);
711 return 0;
712 }
713 #endif
714
715 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags);
716
717 if (va == 0)
718 return (ENOMEM);
719
720 *kvap = (void *)va;
721
722 for (curseg = 0; curseg < nsegs; curseg++) {
723 for (addr = BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr);
724 addr < (BUS_MEM_TO_PHYS(t, segs[curseg].ds_addr)
725 + segs[curseg].ds_len);
726 addr += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) {
727 if (size == 0)
728 panic("_bus_dmamem_map: size botch");
729 /*
730 * If we are mapping nocache, flush the page from
731 * cache before we map it.
732 */
733 if (flags & BUS_DMA_DONTCACHE)
734 dcbf(addr, PAGE_SIZE,
735 curcpu()->ci_ci.dcache_line_size);
736 pmap_kenter_pa(va, addr,
737 VM_PROT_READ | VM_PROT_WRITE,
738 PMAP_WIRED |
739 ((flags & BUS_DMA_DONTCACHE) ? PMAP_NOCACHE : 0));
740 }
741 }
742
743 return (0);
744 }
745
746 /*
747 * Common function for unmapping DMA-safe memory. May be called by
748 * bus-specific DMA memory unmapping functions.
749 */
750 void
_bus_dmamem_unmap(bus_dma_tag_t t,void * kva,size_t size)751 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
752 {
753 vaddr_t va = (vaddr_t) kva;
754
755 #ifdef DIAGNOSTIC
756 if (va & PGOFSET)
757 panic("_bus_dmamem_unmap");
758 #endif
759
760 if (va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS) {
761 size = round_page(size);
762 pmap_kremove(va, size);
763 uvm_km_free(kernel_map, va, size, UVM_KMF_VAONLY);
764 }
765 }
766
767 /*
768 * Common function for mmap(2)'ing DMA-safe memory. May be called by
769 * bus-specific DMA mmap(2)'ing functions.
770 */
771 paddr_t
_bus_dmamem_mmap(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,off_t off,int prot,int flags)772 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off, int prot, int flags)
773 {
774 int i;
775
776 for (i = 0; i < nsegs; i++) {
777 #ifdef DIAGNOSTIC
778 if (off & PGOFSET)
779 panic("_bus_dmamem_mmap: offset unaligned");
780 if (BUS_MEM_TO_PHYS(t, segs[i].ds_addr) & PGOFSET)
781 panic("_bus_dmamem_mmap: segment unaligned");
782 if (segs[i].ds_len & PGOFSET)
783 panic("_bus_dmamem_mmap: segment size not multiple"
784 " of page size");
785 #endif
786 if (off >= segs[i].ds_len) {
787 off -= segs[i].ds_len;
788 continue;
789 }
790
791 return (BUS_MEM_TO_PHYS(t, segs[i].ds_addr) + off);
792 }
793
794 /* Page not found. */
795 return (-1);
796 }
797
798 /*
799 * Allocate physical memory from the given physical address range.
800 * Called by DMA-safe memory allocation methods.
801 */
802 int
_bus_dmamem_alloc_range(bus_dma_tag_t t,bus_size_t size,bus_size_t alignment,bus_size_t boundary,bus_dma_segment_t * segs,int nsegs,int * rsegs,int flags,paddr_t low,paddr_t high)803 _bus_dmamem_alloc_range(
804 bus_dma_tag_t t,
805 bus_size_t size,
806 bus_size_t alignment,
807 bus_size_t boundary,
808 bus_dma_segment_t *segs,
809 int nsegs,
810 int *rsegs,
811 int flags,
812 paddr_t low,
813 paddr_t high)
814 {
815 paddr_t curaddr, lastaddr;
816 struct vm_page *m;
817 struct pglist mlist;
818 int curseg, error;
819
820 /* Always round the size. */
821 size = round_page(size);
822
823 /*
824 * Allocate pages from the VM system.
825 */
826 error = uvm_pglistalloc(size, low, high, alignment, boundary,
827 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0);
828 if (error)
829 return (error);
830
831 /*
832 * Compute the location, size, and number of segments actually
833 * returned by the VM code.
834 */
835 m = mlist.tqh_first;
836 curseg = 0;
837 lastaddr = VM_PAGE_TO_PHYS(m);
838 segs[curseg].ds_addr = PHYS_TO_BUS_MEM(t, lastaddr);
839 segs[curseg].ds_len = PAGE_SIZE;
840 m = m->pageq.queue.tqe_next;
841
842 for (; m != NULL; m = m->pageq.queue.tqe_next) {
843 curaddr = VM_PAGE_TO_PHYS(m);
844 #ifdef DIAGNOSTIC
845 if (curaddr < low || curaddr >= high) {
846 printf("vm_page_alloc_memory returned non-sensical"
847 " address 0x%lx\n", curaddr);
848 panic("_bus_dmamem_alloc_range");
849 }
850 #endif
851 if (curaddr == (lastaddr + PAGE_SIZE))
852 segs[curseg].ds_len += PAGE_SIZE;
853 else {
854 curseg++;
855 segs[curseg].ds_addr = PHYS_TO_BUS_MEM(t, curaddr);
856 segs[curseg].ds_len = PAGE_SIZE;
857 }
858 lastaddr = curaddr;
859 }
860
861 *rsegs = curseg + 1;
862
863 return (0);
864 }
865
866 /*
867 * Generic form of PHYS_TO_BUS_MEM().
868 */
869 bus_addr_t
_bus_dma_phys_to_bus_mem_generic(bus_dma_tag_t t,bus_addr_t addr)870 _bus_dma_phys_to_bus_mem_generic(bus_dma_tag_t t, bus_addr_t addr)
871 {
872
873 return (addr);
874 }
875
876 /*
877 * Generic form of BUS_MEM_TO_PHYS().
878 */
879 bus_addr_t
_bus_dma_bus_mem_to_phys_generic(bus_dma_tag_t t,bus_addr_t addr)880 _bus_dma_bus_mem_to_phys_generic(bus_dma_tag_t t, bus_addr_t addr)
881 {
882
883 return (addr);
884 }
885