1 /* $NetBSD: ttm_bo_vm.c,v 1.10 2015/07/28 01:25:00 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2014 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(0, "$NetBSD: ttm_bo_vm.c,v 1.10 2015/07/28 01:25:00 riastradh Exp $");
34
35 #include <sys/types.h>
36
37 #include <uvm/uvm.h>
38 #include <uvm/uvm_extern.h>
39 #include <uvm/uvm_fault.h>
40
41 #include <linux/bitops.h>
42
43 #include <drm/drm_vma_manager.h>
44
45 #include <ttm/ttm_bo_driver.h>
46
47 static int ttm_bo_uvm_fault_idle(struct ttm_buffer_object *,
48 struct uvm_faultinfo *);
49 static int ttm_bo_uvm_lookup(struct ttm_bo_device *, unsigned long,
50 unsigned long, struct ttm_buffer_object **);
51
52 void
ttm_bo_uvm_reference(struct uvm_object * uobj)53 ttm_bo_uvm_reference(struct uvm_object *uobj)
54 {
55 struct ttm_buffer_object *const bo = container_of(uobj,
56 struct ttm_buffer_object, uvmobj);
57
58 (void)ttm_bo_reference(bo);
59 }
60
61 void
ttm_bo_uvm_detach(struct uvm_object * uobj)62 ttm_bo_uvm_detach(struct uvm_object *uobj)
63 {
64 struct ttm_buffer_object *bo = container_of(uobj,
65 struct ttm_buffer_object, uvmobj);
66
67 ttm_bo_unref(&bo);
68 KASSERT(bo == NULL);
69 }
70
71 int
ttm_bo_uvm_fault(struct uvm_faultinfo * ufi,vaddr_t vaddr,struct vm_page ** pps,int npages,int centeridx,vm_prot_t access_type,int flags)72 ttm_bo_uvm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr,
73 struct vm_page **pps, int npages, int centeridx, vm_prot_t access_type,
74 int flags)
75 {
76 struct uvm_object *const uobj = ufi->entry->object.uvm_obj;
77 struct ttm_buffer_object *const bo = container_of(uobj,
78 struct ttm_buffer_object, uvmobj);
79 struct ttm_bo_device *const bdev = bo->bdev;
80 struct ttm_mem_type_manager *man =
81 &bdev->man[bo->mem.mem_type];
82 union {
83 bus_addr_t base;
84 struct ttm_tt *ttm;
85 } u;
86 size_t size __diagused;
87 voff_t uoffset; /* offset in bytes into bo */
88 unsigned startpage; /* offset in pages into bo */
89 unsigned i;
90 vm_prot_t vm_prot; /* VM_PROT_* */
91 pgprot_t pgprot; /* VM_PROT_* | PMAP_* cacheability flags */
92 unsigned mmapflags;
93 int ret;
94
95 /* Thanks, uvm, but we don't need this lock. */
96 mutex_exit(uobj->vmobjlock);
97
98 /* Copy-on-write mappings make no sense for the graphics aperture. */
99 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) {
100 ret = -EIO;
101 goto out0;
102 }
103
104 /* Try to lock the buffer. */
105 ret = ttm_bo_reserve(bo, true, true, false, NULL);
106 if (ret) {
107 if (ret != -EBUSY)
108 goto out0;
109 /*
110 * It's currently locked. Unlock the fault, wait for
111 * it, and start over.
112 */
113 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL);
114 (void)ttm_bo_wait_unreserved(bo);
115 return -ERESTART;
116 }
117
118 /* drm prime buffers are not mappable. XXX Catch this earlier? */
119 if (bo->ttm && ISSET(bo->ttm->page_flags, TTM_PAGE_FLAG_SG)) {
120 ret = -EINVAL;
121 goto out1;
122 }
123
124 /* Notify the driver of a fault if it wants. */
125 if (bdev->driver->fault_reserve_notify) {
126 ret = (*bdev->driver->fault_reserve_notify)(bo);
127 if (ret) {
128 if (ret == -ERESTART)
129 ret = -EIO;
130 goto out1;
131 }
132 }
133
134 ret = ttm_bo_uvm_fault_idle(bo, ufi);
135 if (ret) {
136 KASSERT(ret == -ERESTART);
137 /* ttm_bo_uvm_fault_idle calls uvmfault_unlockall for us. */
138 ttm_bo_unreserve(bo);
139 /* XXX errno Linux->NetBSD */
140 return -ret;
141 }
142
143 ret = ttm_mem_io_lock(man, true);
144 if (ret) {
145 ret = -EIO;
146 goto out1;
147 }
148 ret = ttm_mem_io_reserve_vm(bo);
149 if (ret) {
150 ret = -EIO;
151 goto out2;
152 }
153
154 vm_prot = ufi->entry->protection;
155 if (bo->mem.bus.is_iomem) {
156 u.base = (bo->mem.bus.base + bo->mem.bus.offset);
157 size = bo->mem.bus.size;
158 pgprot = ttm_io_prot(bo->mem.placement, vm_prot);
159 } else {
160 u.ttm = bo->ttm;
161 size = (bo->ttm->num_pages << PAGE_SHIFT);
162 if (ISSET(bo->mem.placement, TTM_PL_FLAG_CACHED))
163 pgprot = vm_prot;
164 else
165 pgprot = ttm_io_prot(bo->mem.placement, vm_prot);
166 if ((*u.ttm->bdev->driver->ttm_tt_populate)(u.ttm)) {
167 ret = -ENOMEM;
168 goto out2;
169 }
170 }
171
172 KASSERT(ufi->entry->start <= vaddr);
173 KASSERT((ufi->entry->offset & (PAGE_SIZE - 1)) == 0);
174 KASSERT(ufi->entry->offset <= size);
175 KASSERT((vaddr - ufi->entry->start) <= (size - ufi->entry->offset));
176 KASSERT(npages <= ((size - ufi->entry->offset) -
177 (vaddr - ufi->entry->start)));
178 uoffset = (ufi->entry->offset + (vaddr - ufi->entry->start));
179 startpage = (uoffset >> PAGE_SHIFT);
180 for (i = 0; i < npages; i++) {
181 paddr_t paddr;
182
183 /* XXX PGO_ALLPAGES? */
184 if (pps[i] == PGO_DONTCARE)
185 continue;
186 if (bo->mem.bus.is_iomem) {
187 const paddr_t cookie = bus_space_mmap(bdev->memt,
188 u.base, ((startpage + i) << PAGE_SHIFT), vm_prot,
189 0);
190
191 paddr = pmap_phys_address(cookie);
192 mmapflags = pmap_mmap_flags(cookie);
193 } else {
194 paddr = page_to_phys(u.ttm->pages[startpage + i]);
195 mmapflags = 0;
196 }
197 ret = -pmap_enter(ufi->orig_map->pmap, vaddr + i*PAGE_SIZE,
198 paddr, vm_prot, (PMAP_CANFAIL | pgprot | mmapflags));
199 if (ret)
200 goto out3;
201 }
202
203 out3: pmap_update(ufi->orig_map->pmap);
204 out2: ttm_mem_io_unlock(man);
205 out1: ttm_bo_unreserve(bo);
206 out0: uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL);
207 /* XXX errno Linux->NetBSD */
208 return -ret;
209 }
210
211 static int
ttm_bo_uvm_fault_idle(struct ttm_buffer_object * bo,struct uvm_faultinfo * ufi)212 ttm_bo_uvm_fault_idle(struct ttm_buffer_object *bo, struct uvm_faultinfo *ufi)
213 {
214 struct ttm_bo_device *const bdev = bo->bdev;
215 int ret = 0;
216
217 spin_lock(&bdev->fence_lock);
218 if (__predict_true(!test_bit(TTM_BO_PRIV_FLAG_MOVING,
219 &bo->priv_flags)))
220 goto out;
221 if (ttm_bo_wait(bo, false, false, true) == 0)
222 goto out;
223
224 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL);
225 (void)ttm_bo_wait(bo, false, true, false);
226 ret = -ERESTART;
227
228 out: spin_unlock(&bdev->fence_lock);
229 return ret;
230 }
231
232 int
ttm_bo_mmap_object(struct ttm_bo_device * bdev,off_t offset,size_t size,vm_prot_t prot,struct uvm_object ** uobjp,voff_t * uoffsetp,struct file * file)233 ttm_bo_mmap_object(struct ttm_bo_device *bdev, off_t offset, size_t size,
234 vm_prot_t prot, struct uvm_object **uobjp, voff_t *uoffsetp,
235 struct file *file)
236 {
237 const unsigned long startpage = (offset >> PAGE_SHIFT);
238 const unsigned long npages = (size >> PAGE_SHIFT);
239 struct ttm_buffer_object *bo;
240 int ret;
241
242 KASSERT(0 == (offset & (PAGE_SIZE - 1)));
243 KASSERT(0 == (size & (PAGE_SIZE - 1)));
244
245 ret = ttm_bo_uvm_lookup(bdev, startpage, npages, &bo);
246 if (ret)
247 goto fail0;
248 KASSERT(drm_vma_node_start(&bo->vma_node) <= offset);
249 /* XXX Just assert this? */
250 if (__predict_false(bdev->driver->verify_access == NULL)) {
251 ret = -EPERM;
252 goto fail1;
253 }
254 ret = (*bdev->driver->verify_access)(bo, file);
255 if (ret)
256 goto fail1;
257
258 /* Success! */
259 *uobjp = &bo->uvmobj;
260 *uoffsetp = (offset -
261 (drm_vma_node_start(&bo->vma_node) << PAGE_SHIFT));
262 return 0;
263
264 fail1: ttm_bo_unref(&bo);
265 fail0: KASSERT(ret);
266 return ret;
267 }
268
269 static int
ttm_bo_uvm_lookup(struct ttm_bo_device * bdev,unsigned long startpage,unsigned long npages,struct ttm_buffer_object ** bop)270 ttm_bo_uvm_lookup(struct ttm_bo_device *bdev, unsigned long startpage,
271 unsigned long npages, struct ttm_buffer_object **bop)
272 {
273 struct ttm_buffer_object *bo = NULL;
274 struct drm_vma_offset_node *node;
275
276 drm_vma_offset_lock_lookup(&bdev->vma_manager);
277 node = drm_vma_offset_lookup_locked(&bdev->vma_manager, startpage,
278 npages);
279 if (node != NULL) {
280 bo = container_of(node, struct ttm_buffer_object, vma_node);
281 if (!kref_get_unless_zero(&bo->kref))
282 bo = NULL;
283 }
284 drm_vma_offset_unlock_lookup(&bdev->vma_manager);
285
286 if (bo == NULL)
287 return -ENOENT;
288
289 *bop = bo;
290 return 0;
291 }
292