1 /* $NetBSD: xen_bus_dma.c,v 1.22 2010/11/12 10:51:14 njoly Exp $ */ 2 /* NetBSD bus_dma.c,v 1.21 2005/04/16 07:53:35 yamt Exp */ 3 4 /*- 5 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Charles M. Hannum and by Jason R. Thorpe of the Numerical Aerospace 10 * Simulation Facility, NASA Ames Research Center. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: xen_bus_dma.c,v 1.22 2010/11/12 10:51:14 njoly Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/mbuf.h> 41 #include <sys/proc.h> 42 43 #include <machine/bus.h> 44 #include <machine/bus_private.h> 45 46 #include <uvm/uvm.h> 47 48 extern paddr_t avail_end; 49 50 /* Pure 2^n version of get_order */ 51 static inline int get_order(unsigned long size) 52 { 53 int order = -1; 54 size = (size - 1) >> (PAGE_SHIFT - 1); 55 do { 56 size >>= 1; 57 order++; 58 } while (size); 59 return order; 60 } 61 62 static int 63 _xen_alloc_contig(bus_size_t size, bus_size_t alignment, 64 struct pglist *mlistp, int flags, bus_addr_t low, bus_addr_t high) 65 { 66 int order, i; 67 unsigned long npagesreq, npages, mfn; 68 bus_addr_t pa; 69 struct vm_page *pg, *pgnext; 70 int s, error; 71 struct xen_memory_reservation res; 72 73 /* 74 * When requesting a contigous memory region, the hypervisor will 75 * return a memory range aligned on size. 76 * The only way to enforce alignment is to request a memory region 77 * of size max(alignment, size). 78 */ 79 order = max(get_order(size), get_order(alignment)); 80 npages = (1 << order); 81 npagesreq = (size >> PAGE_SHIFT); 82 KASSERT(npages >= npagesreq); 83 84 /* get npages from UVM, and give them back to the hypervisor */ 85 error = uvm_pglistalloc(((psize_t)npages) << PAGE_SHIFT, 86 0, avail_end, 0, 0, mlistp, npages, (flags & BUS_DMA_NOWAIT) == 0); 87 if (error) 88 return (error); 89 90 for (pg = mlistp->tqh_first; pg != NULL; pg = pg->pageq.queue.tqe_next) { 91 pa = VM_PAGE_TO_PHYS(pg); 92 mfn = xpmap_ptom(pa) >> PAGE_SHIFT; 93 xpmap_phys_to_machine_mapping[ 94 (pa - XPMAP_OFFSET) >> PAGE_SHIFT] = INVALID_P2M_ENTRY; 95 xenguest_handle(res.extent_start) = &mfn; 96 res.nr_extents = 1; 97 res.extent_order = 0; 98 res.address_bits = 0; 99 res.domid = DOMID_SELF; 100 error = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &res); 101 if (error != 1) { 102 #ifdef DEBUG 103 printf("xen_alloc_contig: XENMEM_decrease_reservation " 104 "failed: err %d (pa %#" PRIxPADDR " mfn %#lx)\n", 105 error, pa, mfn); 106 #endif 107 xpmap_phys_to_machine_mapping[ 108 (pa - XPMAP_OFFSET) >> PAGE_SHIFT] = mfn; 109 110 error = ENOMEM; 111 goto failed; 112 } 113 } 114 /* Get the new contiguous memory extent */ 115 xenguest_handle(res.extent_start) = &mfn; 116 res.nr_extents = 1; 117 res.extent_order = order; 118 res.address_bits = get_order(high) + PAGE_SHIFT; 119 res.domid = DOMID_SELF; 120 error = HYPERVISOR_memory_op(XENMEM_increase_reservation, &res); 121 if (error != 1) { 122 #ifdef DEBUG 123 printf("xen_alloc_contig: XENMEM_increase_reservation " 124 "failed: %d (order %d address_bits %d)\n", 125 error, order, res.address_bits); 126 #endif 127 error = ENOMEM; 128 pg = NULL; 129 goto failed; 130 } 131 s = splvm(); 132 /* Map the new extent in place of the old pages */ 133 for (pg = mlistp->tqh_first, i = 0; pg != NULL; pg = pgnext, i++) { 134 pgnext = pg->pageq.queue.tqe_next; 135 pa = VM_PAGE_TO_PHYS(pg); 136 xpmap_phys_to_machine_mapping[ 137 (pa - XPMAP_OFFSET) >> PAGE_SHIFT] = mfn+i; 138 xpq_queue_machphys_update(((paddr_t)(mfn+i)) << PAGE_SHIFT, pa); 139 /* while here, give extra pages back to UVM */ 140 if (i >= npagesreq) { 141 TAILQ_REMOVE(mlistp, pg, pageq.queue); 142 uvm_pagefree(pg); 143 } 144 } 145 /* Flush updates through and flush the TLB */ 146 xpq_queue_tlb_flush(); 147 splx(s); 148 return 0; 149 150 failed: 151 /* 152 * Attempt to recover from a failed decrease or increase reservation: 153 * if decrease_reservation failed, we don't have given all pages 154 * back to Xen; give them back to UVM, and get the missing pages 155 * from Xen. 156 * if increase_reservation failed, we expect pg to be NULL and we just 157 * get back the missing pages from Xen one by one. 158 */ 159 /* give back remaining pages to UVM */ 160 for (; pg != NULL; pg = pgnext) { 161 pgnext = pg->pageq.queue.tqe_next; 162 TAILQ_REMOVE(mlistp, pg, pageq.queue); 163 uvm_pagefree(pg); 164 } 165 /* remplace the pages that we already gave to Xen */ 166 s = splvm(); 167 for (pg = mlistp->tqh_first; pg != NULL; pg = pgnext) { 168 pgnext = pg->pageq.queue.tqe_next; 169 xenguest_handle(res.extent_start) = &mfn; 170 res.nr_extents = 1; 171 res.extent_order = 0; 172 res.address_bits = 32; 173 res.domid = DOMID_SELF; 174 if (HYPERVISOR_memory_op(XENMEM_increase_reservation, &res) 175 < 0) { 176 printf("xen_alloc_contig: recovery " 177 "XENMEM_increase_reservation failed!\n"); 178 break; 179 } 180 pa = VM_PAGE_TO_PHYS(pg); 181 xpmap_phys_to_machine_mapping[ 182 (pa - XPMAP_OFFSET) >> PAGE_SHIFT] = mfn; 183 xpq_queue_machphys_update(((paddr_t)mfn) << PAGE_SHIFT, pa); 184 TAILQ_REMOVE(mlistp, pg, pageq.queue); 185 uvm_pagefree(pg); 186 } 187 /* Flush updates through and flush the TLB */ 188 xpq_queue_tlb_flush(); 189 splx(s); 190 return error; 191 } 192 193 194 /* 195 * Allocate physical memory from the given physical address range. 196 * Called by DMA-safe memory allocation methods. 197 * We need our own version to deal with physical vs machine addresses. 198 */ 199 int 200 _xen_bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, 201 bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, 202 int nsegs, int *rsegs, int flags, bus_addr_t low, bus_addr_t high) 203 { 204 bus_addr_t curaddr, lastaddr; 205 struct vm_page *m; 206 struct pglist mlist; 207 int curseg, error; 208 int doingrealloc = 0; 209 bus_size_t uboundary; 210 211 /* Always round the size. */ 212 size = round_page(size); 213 214 KASSERT((alignment & (alignment - 1)) == 0); 215 KASSERT((boundary & (boundary - 1)) == 0); 216 KASSERT(boundary >= PAGE_SIZE || boundary == 0); 217 218 if (alignment < PAGE_SIZE) 219 alignment = PAGE_SIZE; 220 221 /* 222 * Allocate pages from the VM system. 223 * We accept boundaries < size, splitting in multiple segments 224 * if needed. uvm_pglistalloc does not, so compute an appropriate 225 * boundary: next power of 2 >= size 226 */ 227 if (boundary == 0) 228 uboundary = 0; 229 else { 230 uboundary = boundary; 231 while (uboundary < size) 232 uboundary = uboundary << 1; 233 } 234 error = uvm_pglistalloc(size, 0, avail_end, alignment, uboundary, 235 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0); 236 if (error) 237 return (error); 238 again: 239 240 /* 241 * Compute the location, size, and number of segments actually 242 * returned by the VM code. 243 */ 244 m = mlist.tqh_first; 245 curseg = 0; 246 curaddr = lastaddr = segs[curseg].ds_addr = _BUS_VM_PAGE_TO_BUS(m); 247 if (curaddr < low || curaddr >= high) 248 goto badaddr; 249 segs[curseg].ds_len = PAGE_SIZE; 250 m = m->pageq.queue.tqe_next; 251 if ((segs[curseg].ds_addr & (alignment - 1)) != 0) 252 goto dorealloc; 253 254 for (; m != NULL; m = m->pageq.queue.tqe_next) { 255 curaddr = _BUS_VM_PAGE_TO_BUS(m); 256 if (curaddr < low || curaddr >= high) 257 goto badaddr; 258 if (curaddr == (lastaddr + PAGE_SIZE) && 259 (lastaddr & boundary) == (curaddr & boundary)) { 260 segs[curseg].ds_len += PAGE_SIZE; 261 } else { 262 curseg++; 263 if (curseg >= nsegs || 264 (curaddr & (alignment - 1)) != 0) { 265 if (doingrealloc) 266 return EFBIG; 267 else 268 goto dorealloc; 269 } 270 segs[curseg].ds_addr = curaddr; 271 segs[curseg].ds_len = PAGE_SIZE; 272 } 273 lastaddr = curaddr; 274 } 275 276 *rsegs = curseg + 1; 277 return (0); 278 279 badaddr: 280 if (doingrealloc == 0) 281 goto dorealloc; 282 if (curaddr < low) { 283 /* no way to enforce this */ 284 printf("_xen_bus_dmamem_alloc_range: no way to " 285 "enforce address range (0x%" PRIx64 " - 0x%" PRIx64 ")\n", 286 (uint64_t)low, (uint64_t)high); 287 uvm_pglistfree(&mlist); 288 return EINVAL; 289 } 290 printf("xen_bus_dmamem_alloc_range: " 291 "curraddr=0x%lx > high=0x%lx\n", 292 (u_long)curaddr, (u_long)high); 293 panic("xen_bus_dmamem_alloc_range 1"); 294 dorealloc: 295 if (doingrealloc == 1) 296 panic("_xen_bus_dmamem_alloc_range: " 297 "xen_alloc_contig returned " 298 "too much segments"); 299 doingrealloc = 1; 300 /* 301 * Too much segments, or memory doesn't fit 302 * constraints. Free this memory and 303 * get a contigous segment from the hypervisor. 304 */ 305 uvm_pglistfree(&mlist); 306 for (curseg = 0; curseg < nsegs; curseg++) { 307 segs[curseg].ds_addr = 0; 308 segs[curseg].ds_len = 0; 309 } 310 error = _xen_alloc_contig(size, alignment, 311 &mlist, flags, low, high); 312 if (error) 313 return error; 314 goto again; 315 } 316