1 /* $NetBSD: bus_dma.c,v 1.31 2022/07/26 20:08:55 andvar Exp $ */
2
3 /*-
4 * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9 * NASA Ames Research Center.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 */
32
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.31 2022/07/26 20:08:55 andvar Exp $");
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/mbuf.h>
39 #include <sys/device.h>
40 #include <sys/proc.h>
41 #include <sys/kmem.h>
42
43 #include <uvm/uvm_extern.h>
44
45 #define _MIPSCO_BUS_DMA_PRIVATE
46 #include <machine/bus.h>
47
48 #include <dev/bus_dma/bus_dmamem_common.h>
49
50 #include <mips/cache.h>
51
52 paddr_t kvtophys(vaddr_t); /* XXX */
53
54 static int _bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t,
55 void *, bus_size_t, struct vmspace *, int, paddr_t *,
56 int *, int);
57
58 void
_bus_dma_tag_init(bus_dma_tag_t t)59 _bus_dma_tag_init(bus_dma_tag_t t)
60 {
61 t->dma_offset = 0;
62
63 t->_dmamap_create = _bus_dmamap_create;
64 t->_dmamap_destroy = _bus_dmamap_destroy;
65 t->_dmamap_load = _bus_dmamap_load;
66 t->_dmamap_load_mbuf = _bus_dmamap_load_mbuf;
67 t->_dmamap_load_uio = _bus_dmamap_load_uio;
68 t->_dmamap_load_raw = _bus_dmamap_load_raw;
69 t->_dmamap_unload = _bus_dmamap_unload;
70 t->_dmamap_sync = _bus_dmamap_sync;
71 t->_dmamem_alloc = _bus_dmamem_alloc;
72 t->_dmamem_free = _bus_dmamem_free;
73 t->_dmamem_map = _bus_dmamem_map;
74 t->_dmamem_unmap = _bus_dmamem_unmap;
75 t->_dmamem_mmap = _bus_dmamem_mmap;
76 }
77
78 static size_t
_bus_dmamap_mapsize(int const nsegments)79 _bus_dmamap_mapsize(int const nsegments)
80 {
81 KASSERT(nsegments > 0);
82 return sizeof(struct mipsco_bus_dmamap) +
83 (sizeof(bus_dma_segment_t) * (nsegments - 1));
84 }
85
86 /*
87 * Common function for DMA map creation. May be called by bus-specific
88 * DMA map creation functions.
89 */
90 int
_bus_dmamap_create(bus_dma_tag_t t,bus_size_t size,int nsegments,bus_size_t maxsegsz,bus_size_t boundary,int flags,bus_dmamap_t * dmamp)91 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
92 {
93 struct mipsco_bus_dmamap *map;
94 void *mapstore;
95
96 /*
97 * Allocate and initialize the DMA map. The end of the map
98 * is a variable-sized array of segments, so we allocate enough
99 * room for them in one shot.
100 *
101 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation
102 * of ALLOCNOW notifies others that we've reserved these resources,
103 * and they are not to be freed.
104 *
105 * The bus_dmamap_t includes one bus_dma_segment_t, hence
106 * the (nsegments - 1).
107 */
108 if ((mapstore = kmem_zalloc(_bus_dmamap_mapsize(nsegments),
109 (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP)) == NULL)
110 return (ENOMEM);
111
112 map = (struct mipsco_bus_dmamap *)mapstore;
113 map->_dm_size = size;
114 map->_dm_segcnt = nsegments;
115 map->_dm_maxmaxsegsz = maxsegsz;
116 map->_dm_boundary = boundary;
117 map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT);
118 map->dm_maxsegsz = maxsegsz;
119 map->dm_mapsize = 0; /* no valid mappings */
120 map->dm_nsegs = 0;
121
122 *dmamp = map;
123 return (0);
124 }
125
126 /*
127 * Common function for DMA map destruction. May be called by bus-specific
128 * DMA map destruction functions.
129 */
130 void
_bus_dmamap_destroy(bus_dma_tag_t t,bus_dmamap_t map)131 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
132 {
133
134 kmem_free(map, _bus_dmamap_mapsize(map->_dm_segcnt));
135 }
136
137 /*
138 * Utility function to load a linear buffer. lastaddrp holds state
139 * between invocations (for multiple-buffer loads). segp contains
140 * the starting segment on entrance, and the ending segment on exit.
141 * first indicates if this is the first invocation of this function.
142 */
143 static int
_bus_dmamap_load_buffer(bus_dma_tag_t t,bus_dmamap_t map,void * buf,bus_size_t buflen,struct vmspace * vm,int flags,paddr_t * lastaddrp,int * segp,int first)144 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct vmspace *vm, int flags, paddr_t *lastaddrp, int *segp, int first)
145 {
146 bus_size_t sgsize;
147 bus_addr_t baddr, bmask;
148 paddr_t curaddr, lastaddr;
149 vaddr_t vaddr = (vaddr_t)buf;
150 int seg;
151
152 lastaddr = *lastaddrp;
153 bmask = ~(map->_dm_boundary - 1);
154
155 for (seg = *segp; buflen > 0 ; ) {
156 /*
157 * Get the physical address for this segment.
158 */
159 if (!VMSPACE_IS_KERNEL_P(vm))
160 (void) pmap_extract(vm_map_pmap(&vm->vm_map),
161 vaddr, &curaddr);
162 else
163 curaddr = kvtophys(vaddr);
164
165 /*
166 * Compute the segment size, and adjust counts.
167 */
168 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
169 if (buflen < sgsize)
170 sgsize = buflen;
171
172 /*
173 * Make sure we don't cross any boundaries.
174 */
175 if (map->_dm_boundary > 0) {
176 baddr = (curaddr + map->_dm_boundary) & bmask;
177 if (sgsize > (baddr - curaddr))
178 sgsize = (baddr - curaddr);
179 }
180
181 /*
182 * Insert chunk into a segment, coalescing with
183 * the previous segment if possible.
184 */
185 if (first) {
186 map->dm_segs[seg].ds_addr = curaddr + t->dma_offset;
187 map->dm_segs[seg].ds_len = sgsize;
188 map->dm_segs[seg]._ds_vaddr = vaddr;
189 map->dm_segs[seg]._ds_paddr = curaddr;
190 first = 0;
191 } else {
192 if (curaddr == lastaddr &&
193 (map->dm_segs[seg].ds_len + sgsize) <=
194 map->dm_maxsegsz &&
195 (map->_dm_boundary == 0 ||
196 (map->dm_segs[seg]._ds_paddr & bmask) ==
197 (curaddr & bmask)))
198 map->dm_segs[seg].ds_len += sgsize;
199 else {
200 if (++seg >= map->_dm_segcnt)
201 break;
202 map->dm_segs[seg].ds_addr =
203 curaddr + t->dma_offset;
204 map->dm_segs[seg].ds_len = sgsize;
205 map->dm_segs[seg]._ds_vaddr = vaddr;
206 map->dm_segs[seg]._ds_paddr = curaddr;
207 }
208 }
209
210 lastaddr = curaddr + sgsize;
211 vaddr += sgsize;
212 buflen -= sgsize;
213 }
214
215 *segp = seg;
216 *lastaddrp = lastaddr;
217
218 /*
219 * Did we fit?
220 */
221 if (buflen != 0)
222 return (EFBIG); /* XXX better return value here? */
223
224 return (0);
225 }
226
227 /*
228 * Common function for loading a direct-mapped DMA map with a linear
229 * buffer.
230 */
231 int
_bus_dmamap_load(bus_dma_tag_t t,bus_dmamap_t map,void * buf,bus_size_t buflen,struct proc * p,int flags)232 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, bus_size_t buflen, struct proc *p, int flags)
233 {
234 paddr_t lastaddr;
235 int seg, error;
236 struct vmspace *vm;
237
238 /*
239 * Make sure that on error condition we return "no valid mappings".
240 */
241 map->dm_mapsize = 0;
242 map->dm_nsegs = 0;
243 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
244
245 if (buflen > map->_dm_size)
246 return (EINVAL);
247
248 if (p != NULL) {
249 vm = p->p_vmspace;
250 } else {
251 vm = vmspace_kernel();
252 }
253
254 seg = 0;
255 error = _bus_dmamap_load_buffer(t, map, buf, buflen,
256 vm, flags, &lastaddr, &seg, 1);
257 if (error == 0) {
258 map->dm_mapsize = buflen;
259 map->dm_nsegs = seg + 1;
260
261 /*
262 * For linear buffers, we support marking the mapping
263 * as COHERENT.
264 *
265 * XXX Check TLB entries for cache-inhibit bits?
266 */
267 if (buf >= (void *)MIPS_KSEG1_START &&
268 buf < (void *)MIPS_KSEG2_START)
269 map->_dm_flags |= MIPSCO_DMAMAP_COHERENT;
270 }
271 return (error);
272 }
273
274 /*
275 * Like _bus_dmamap_load(), but for mbufs.
276 */
277 int
_bus_dmamap_load_mbuf(bus_dma_tag_t t,bus_dmamap_t map,struct mbuf * m0,int flags)278 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, int flags)
279 {
280 paddr_t lastaddr;
281 int seg, error, first;
282 struct mbuf *m;
283
284 /*
285 * Make sure that on error condition we return "no valid mappings."
286 */
287 map->dm_mapsize = 0;
288 map->dm_nsegs = 0;
289 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
290
291 #ifdef DIAGNOSTIC
292 if ((m0->m_flags & M_PKTHDR) == 0)
293 panic("_bus_dmamap_load_mbuf: no packet header");
294 #endif
295
296 if (m0->m_pkthdr.len > map->_dm_size)
297 return (EINVAL);
298
299 first = 1;
300 seg = 0;
301 error = 0;
302 for (m = m0; m != NULL && error == 0; m = m->m_next) {
303 if (m->m_len == 0)
304 continue;
305 error = _bus_dmamap_load_buffer(t, map, m->m_data, m->m_len,
306 vmspace_kernel(), flags, &lastaddr, &seg, first);
307 first = 0;
308 }
309 if (error == 0) {
310 map->dm_mapsize = m0->m_pkthdr.len;
311 map->dm_nsegs = seg + 1;
312 }
313 return (error);
314 }
315
316 /*
317 * Like _bus_dmamap_load(), but for uios.
318 */
319 int
_bus_dmamap_load_uio(bus_dma_tag_t t,bus_dmamap_t map,struct uio * uio,int flags)320 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, int flags)
321 {
322 paddr_t lastaddr;
323 int seg, i, error, first;
324 bus_size_t minlen, resid;
325 struct iovec *iov;
326 void *addr;
327
328 /*
329 * Make sure that on error condition we return "no valid mappings."
330 */
331 map->dm_mapsize = 0;
332 map->dm_nsegs = 0;
333 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
334
335 resid = uio->uio_resid;
336 iov = uio->uio_iov;
337
338 first = 1;
339 seg = 0;
340 error = 0;
341 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
342 /*
343 * Now at the first iovec to load. Load each iovec
344 * until we have exhausted the residual count.
345 */
346 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
347 addr = (void *)iov[i].iov_base;
348
349 error = _bus_dmamap_load_buffer(t, map, addr, minlen,
350 uio->uio_vmspace, flags, &lastaddr, &seg, first);
351 first = 0;
352
353 resid -= minlen;
354 }
355 if (error == 0) {
356 map->dm_mapsize = uio->uio_resid;
357 map->dm_nsegs = seg + 1;
358 }
359 return (error);
360 }
361
362 /*
363 * Like _bus_dmamap_load(), but for raw memory.
364 */
365 int
_bus_dmamap_load_raw(bus_dma_tag_t t,bus_dmamap_t map,bus_dma_segment_t * segs,int nsegs,bus_size_t size,int flags)366 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
367 {
368 panic("_bus_dmamap_load_raw: not implemented");
369 }
370
371 /*
372 * Common function for unloading a DMA map. May be called by
373 * chipset-specific DMA map unload functions.
374 */
375 void
_bus_dmamap_unload(bus_dma_tag_t t,bus_dmamap_t map)376 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
377 {
378
379 /*
380 * No resources to free; just mark the mappings as
381 * invalid.
382 */
383 map->dm_maxsegsz = map->_dm_maxmaxsegsz;
384 map->dm_mapsize = 0;
385 map->dm_nsegs = 0;
386 map->_dm_flags &= ~MIPSCO_DMAMAP_COHERENT;
387 }
388
389 /*
390 * Common function for MIPS1 DMA map synchronization. May be called
391 * by chipset-specific DMA map synchronization functions.
392 *
393 * This is the R3000 version.
394 */
395 void
_bus_dmamap_sync(bus_dma_tag_t t,bus_dmamap_t map,bus_addr_t offset,bus_size_t len,int ops)396 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, bus_size_t len, int ops)
397 {
398 bus_size_t minlen;
399 bus_addr_t addr;
400 int i;
401
402 #ifdef DIAGNOSTIC
403 /*
404 * Mixing PRE and POST operations is not allowed.
405 */
406 if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 &&
407 (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0)
408 panic("_bus_dmamap_sync: mix PRE and POST");
409
410 if (offset >= map->dm_mapsize)
411 panic("_bus_dmamap_sync: bad offset %lu (map size is %lu)",
412 offset, map->dm_mapsize);
413 if (len == 0 || (offset + len) > map->dm_mapsize)
414 panic("_bus_dmamap_sync: bad length");
415 #endif
416
417 /*
418 * The R3000 cache is write-through. Therefore, we only need
419 * to drain the write buffer on PREWRITE. The cache is not
420 * coherent, however, so we need to invalidate the data cache
421 * on POSTREAD.
422 *
423 * PREREAD and POSTWRITE are noops.
424 */
425
426 if (ops & BUS_DMASYNC_PREWRITE) {
427 /*
428 * Flush the write buffer.
429 */
430 wbflush();
431 }
432
433 /*
434 * If we're not doing a POSTREAD, nothing more to do.
435 */
436 if ((ops & BUS_DMASYNC_POSTREAD) == 0)
437 return;
438
439 /*
440 * If the mapping is of COHERENT DMA-safe memory, no cache
441 * flush is necessary.
442 */
443 if (map->_dm_flags & MIPSCO_DMAMAP_COHERENT)
444 return;
445
446 /*
447 * If we are going to hit something as large or larger
448 * than the entire data cache, just nail the whole thing.
449 *
450 * NOTE: Even though this is `wbinv_all', since the cache is
451 * write-through, it just invalidates it.
452 */
453 if (len >= mips_cache_info.mci_pdcache_size) {
454 mips_dcache_wbinv_all();
455 return;
456 }
457
458 for (i = 0; i < map->dm_nsegs && len != 0; i++) {
459 /* Find the beginning segment. */
460 if (offset >= map->dm_segs[i].ds_len) {
461 offset -= map->dm_segs[i].ds_len;
462 continue;
463 }
464
465 /*
466 * Now at the first segment to sync; nail
467 * each segment until we have exhausted the
468 * length.
469 */
470 minlen = len < map->dm_segs[i].ds_len - offset ?
471 len : map->dm_segs[i].ds_len - offset;
472
473 addr = map->dm_segs[i].ds_addr;
474
475 #ifdef BUS_DMA_DEBUG
476 printf("bus_dmamap_sync: flushing segment %d "
477 "(0x%lx..0x%lx) ...", i, addr + offset,
478 addr + offset + minlen - 1);
479 #endif
480 mips_dcache_inv_range(
481 MIPS_PHYS_TO_KSEG0(addr + offset), minlen);
482 #ifdef BUS_DMA_DEBUG
483 printf("\n");
484 #endif
485 offset = 0;
486 len -= minlen;
487 }
488 }
489
490 /*
491 * Common function for DMA-safe memory allocation. May be called
492 * by bus-specific DMA memory allocation functions.
493 */
494 int
_bus_dmamem_alloc(bus_dma_tag_t t,bus_size_t size,bus_size_t alignment,bus_size_t boundary,bus_dma_segment_t * segs,int nsegs,int * rsegs,int flags)495 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, int flags)
496 {
497
498 return _bus_dmamem_alloc_range(t, size, alignment, boundary,
499 segs, nsegs, rsegs, flags, pmap_limits.avail_start,
500 trunc_page(pmap_limits.avail_end));
501 }
502
503 /*
504 * Allocate physical memory from the given physical address range.
505 * Called by DMA-safe memory allocation methods.
506 */
507 int
_bus_dmamem_alloc_range(bus_dma_tag_t t,bus_size_t size,bus_size_t alignment,bus_size_t boundary,bus_dma_segment_t * segs,int nsegs,int * rsegs,int flags,paddr_t low,paddr_t high)508 _bus_dmamem_alloc_range(
509 bus_dma_tag_t t,
510 bus_size_t size,
511 bus_size_t alignment,
512 bus_size_t boundary,
513 bus_dma_segment_t *segs,
514 int nsegs,
515 int *rsegs,
516 int flags,
517 paddr_t low,
518 paddr_t high)
519 {
520
521 return (_bus_dmamem_alloc_range_common(t, size, alignment, boundary,
522 segs, nsegs, rsegs, flags,
523 low, high));
524 }
525
526 /*
527 * Common function for freeing DMA-safe memory. May be called by
528 * bus-specific DMA memory free functions.
529 */
530 void
_bus_dmamem_free(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs)531 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
532 {
533
534 _bus_dmamem_free_common(t, segs, nsegs);
535 }
536
537 /*
538 * Common function for mapping DMA-safe memory. May be called by
539 * bus-specific DMA memory map functions.
540 */
541 int
_bus_dmamem_map(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,size_t size,void ** kvap,int flags)542 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, size_t size, void **kvap, int flags)
543 {
544
545 /*
546 * If we're only mapping 1 segment, use KSEG0 or KSEG1, to avoid
547 * TLB thrashing.
548 */
549 if (nsegs == 1) {
550 if (flags & BUS_DMA_COHERENT)
551 *kvap = (void *)MIPS_PHYS_TO_KSEG1(segs[0]._ds_paddr);
552 else
553 *kvap = (void *)MIPS_PHYS_TO_KSEG0(segs[0]._ds_paddr);
554 return (0);
555 }
556
557 /* XXX BUS_DMA_COHERENT */
558 return (_bus_dmamem_map_common(t, segs, nsegs, size, kvap, flags, 0));
559 }
560
561 /*
562 * Common function for unmapping DMA-safe memory. May be called by
563 * bus-specific DMA memory unmapping functions.
564 */
565 void
_bus_dmamem_unmap(bus_dma_tag_t t,void * kva,size_t size)566 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
567 {
568
569 /*
570 * Nothing to do if we mapped it with KSEG0 or KSEG1 (i.e.
571 * not in KSEG2).
572 */
573 if (kva >= (void *)MIPS_KSEG0_START &&
574 kva < (void *)MIPS_KSEG2_START)
575 return;
576
577 _bus_dmamem_unmap_common(t, kva, size);
578 }
579
580 /*
581 * Common function for mmap(2)'ing DMA-safe memory. May be called by
582 * bus-specific DMA mmap(2)'ing functions.
583 */
584 paddr_t
_bus_dmamem_mmap(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,off_t off,int prot,int flags)585 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, off_t off, int prot, int flags)
586 {
587 bus_addr_t rv;
588
589 rv = _bus_dmamem_mmap_common(t, segs, nsegs, off, prot, flags);
590 if (rv == (bus_addr_t)-1)
591 return (-1);
592
593 return (mips_btop((char *)rv));
594 }
595