1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7 #include <linux/highmem.h>
8 #include <linux/shmem_fs.h>
9 #include <linux/swap.h>
10
11 #include <drm/drm_cache.h>
12 #include <drm/drm_legacy.h> /* for drm_dmamem_alloc() */
13
14 #include "gt/intel_gt.h"
15 #include "i915_drv.h"
16 #include "i915_gem_object.h"
17 #include "i915_gem_region.h"
18 #include "i915_gem_tiling.h"
19 #include "i915_scatterlist.h"
20
i915_gem_object_get_pages_phys(struct drm_i915_gem_object * obj)21 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
22 {
23 #ifdef __linux__
24 struct address_space *mapping = obj->base.filp->f_mapping;
25 #else
26 struct drm_dmamem *dmah;
27 int flags = 0;
28 #endif
29 struct drm_i915_private *i915 = to_i915(obj->base.dev);
30 struct scatterlist *sg;
31 struct sg_table *st;
32 dma_addr_t dma;
33 void *vaddr;
34 void *dst;
35 int i;
36
37 /* Contiguous chunk, with a single scatterlist element */
38 if (overflows_type(obj->base.size, sg->length))
39 return -E2BIG;
40
41 if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
42 return -EINVAL;
43
44 /*
45 * Always aligning to the object size, allows a single allocation
46 * to handle all possible callers, and given typical object sizes,
47 * the alignment of the buddy allocation will naturally match.
48 */
49 #ifdef __linux__
50 vaddr = dma_alloc_coherent(obj->base.dev->dev,
51 roundup_pow_of_two(obj->base.size),
52 &dma, GFP_KERNEL);
53 if (!vaddr)
54 return -ENOMEM;
55 #else
56 dmah = drm_dmamem_alloc(i915->dmat,
57 roundup_pow_of_two(obj->base.size),
58 PAGE_SIZE, 1,
59 roundup_pow_of_two(obj->base.size), flags, 0);
60 if (dmah == NULL)
61 return -ENOMEM;
62 dma = dmah->map->dm_segs[0].ds_addr;
63 vaddr = dmah->kva;
64 #endif
65
66 st = kmalloc(sizeof(*st), GFP_KERNEL);
67 if (!st)
68 goto err_pci;
69
70 if (sg_alloc_table(st, 1, GFP_KERNEL))
71 goto err_st;
72
73 sg = st->sgl;
74 sg->offset = 0;
75 sg->length = obj->base.size;
76
77 #ifdef __linux__
78 sg_assign_page(sg, (struct page *)vaddr);
79 #else
80 sg_assign_page(sg, (struct vm_page *)dmah);
81 #endif
82 sg_dma_address(sg) = dma;
83 sg_dma_len(sg) = obj->base.size;
84
85 dst = vaddr;
86 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
87 struct vm_page *page;
88 void *src;
89
90 #ifdef __linux__
91 page = shmem_read_mapping_page(mapping, i);
92 if (IS_ERR(page))
93 goto err_st;
94 #else
95 struct pglist plist;
96 TAILQ_INIT(&plist);
97 if (uvm_obj_wire(obj->base.uao, i * PAGE_SIZE,
98 (i + 1) * PAGE_SIZE, &plist))
99 goto err_st;
100 page = TAILQ_FIRST(&plist);
101 #endif
102
103 src = kmap_atomic(page);
104 memcpy(dst, src, PAGE_SIZE);
105 drm_clflush_virt_range(dst, PAGE_SIZE);
106 kunmap_atomic(src);
107
108 #ifdef __linux__
109 put_page(page);
110 #else
111 uvm_obj_unwire(obj->base.uao, i * PAGE_SIZE,
112 (i + 1) * PAGE_SIZE);
113 #endif
114 dst += PAGE_SIZE;
115 }
116
117 intel_gt_chipset_flush(to_gt(i915));
118
119 /* We're no longer struct page backed */
120 obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE;
121 __i915_gem_object_set_pages(obj, st);
122
123 return 0;
124
125 err_st:
126 kfree(st);
127 err_pci:
128 #ifdef __linux__
129 dma_free_coherent(obj->base.dev->dev,
130 roundup_pow_of_two(obj->base.size),
131 vaddr, dma);
132 #else
133 drm_dmamem_free(i915->dmat, dmah);
134 #endif
135 return -ENOMEM;
136 }
137
138 void
i915_gem_object_put_pages_phys(struct drm_i915_gem_object * obj,struct sg_table * pages)139 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
140 struct sg_table *pages)
141 {
142 dma_addr_t dma = sg_dma_address(pages->sgl);
143 #ifdef __linux__
144 void *vaddr = sg_page(pages->sgl);
145 #else
146 struct drm_dmamem *dmah = (void *)sg_page(pages->sgl);
147 void *vaddr = dmah->kva;
148 struct drm_i915_private *i915 = to_i915(obj->base.dev);
149 #endif
150
151 __i915_gem_object_release_shmem(obj, pages, false);
152
153 if (obj->mm.dirty) {
154 #ifdef __linux__
155 struct address_space *mapping = obj->base.filp->f_mapping;
156 #endif
157 void *src = vaddr;
158 int i;
159
160 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
161 struct vm_page *page;
162 char *dst;
163
164 #ifdef __linux__
165 page = shmem_read_mapping_page(mapping, i);
166 if (IS_ERR(page))
167 continue;
168 #else
169 struct pglist plist;
170 TAILQ_INIT(&plist);
171 if (uvm_obj_wire(obj->base.uao, i * PAGE_SIZE,
172 (i + 1) * PAGE_SIZE, &plist))
173 continue;
174 page = TAILQ_FIRST(&plist);
175 #endif
176
177 dst = kmap_atomic(page);
178 drm_clflush_virt_range(src, PAGE_SIZE);
179 memcpy(dst, src, PAGE_SIZE);
180 kunmap_atomic(dst);
181
182 set_page_dirty(page);
183 #ifdef __linux__
184 if (obj->mm.madv == I915_MADV_WILLNEED)
185 mark_page_accessed(page);
186 put_page(page);
187 #else
188 uvm_obj_unwire(obj->base.uao, i * PAGE_SIZE,
189 (i + 1) * PAGE_SIZE);
190 #endif
191
192 src += PAGE_SIZE;
193 }
194 obj->mm.dirty = false;
195 }
196
197 sg_free_table(pages);
198 kfree(pages);
199
200 #ifdef __linux__
201 dma_free_coherent(obj->base.dev->dev,
202 roundup_pow_of_two(obj->base.size),
203 vaddr, dma);
204 #else
205 drm_dmamem_free(i915->dmat, dmah);
206 #endif
207 }
208
i915_gem_object_pwrite_phys(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pwrite * args)209 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
210 const struct drm_i915_gem_pwrite *args)
211 {
212 #ifdef __linux__
213 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
214 #else
215 struct drm_dmamem *dmah = (void *)sg_page(obj->mm.pages->sgl);
216 void *vaddr = dmah->kva + args->offset;
217 #endif
218 char __user *user_data = u64_to_user_ptr(args->data_ptr);
219 struct drm_i915_private *i915 = to_i915(obj->base.dev);
220 int err;
221
222 err = i915_gem_object_wait(obj,
223 I915_WAIT_INTERRUPTIBLE |
224 I915_WAIT_ALL,
225 MAX_SCHEDULE_TIMEOUT);
226 if (err)
227 return err;
228
229 /*
230 * We manually control the domain here and pretend that it
231 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
232 */
233 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
234
235 if (copy_from_user(vaddr, user_data, args->size))
236 return -EFAULT;
237
238 drm_clflush_virt_range(vaddr, args->size);
239 intel_gt_chipset_flush(to_gt(i915));
240
241 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
242 return 0;
243 }
244
i915_gem_object_pread_phys(struct drm_i915_gem_object * obj,const struct drm_i915_gem_pread * args)245 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj,
246 const struct drm_i915_gem_pread *args)
247 {
248 #ifdef __linux__
249 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
250 #else
251 struct drm_dmamem *dmah = (void *)sg_page(obj->mm.pages->sgl);
252 void *vaddr = dmah->kva + args->offset;
253 #endif
254 char __user *user_data = u64_to_user_ptr(args->data_ptr);
255 int err;
256
257 err = i915_gem_object_wait(obj,
258 I915_WAIT_INTERRUPTIBLE,
259 MAX_SCHEDULE_TIMEOUT);
260 if (err)
261 return err;
262
263 drm_clflush_virt_range(vaddr, args->size);
264 if (copy_to_user(user_data, vaddr, args->size))
265 return -EFAULT;
266
267 return 0;
268 }
269
i915_gem_object_shmem_to_phys(struct drm_i915_gem_object * obj)270 static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj)
271 {
272 struct sg_table *pages;
273 int err;
274
275 pages = __i915_gem_object_unset_pages(obj);
276
277 err = i915_gem_object_get_pages_phys(obj);
278 if (err)
279 goto err_xfer;
280
281 /* Perma-pin (until release) the physical set of pages */
282 __i915_gem_object_pin_pages(obj);
283
284 if (!IS_ERR_OR_NULL(pages))
285 i915_gem_object_put_pages_shmem(obj, pages);
286
287 i915_gem_object_release_memory_region(obj);
288 return 0;
289
290 err_xfer:
291 if (!IS_ERR_OR_NULL(pages))
292 __i915_gem_object_set_pages(obj, pages);
293 return err;
294 }
295
i915_gem_object_attach_phys(struct drm_i915_gem_object * obj,int align)296 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
297 {
298 int err;
299
300 assert_object_held(obj);
301
302 if (align > obj->base.size)
303 return -EINVAL;
304
305 if (!i915_gem_object_is_shmem(obj))
306 return -EINVAL;
307
308 if (!i915_gem_object_has_struct_page(obj))
309 return 0;
310
311 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
312 if (err)
313 return err;
314
315 if (obj->mm.madv != I915_MADV_WILLNEED)
316 return -EFAULT;
317
318 if (i915_gem_object_has_tiling_quirk(obj))
319 return -EFAULT;
320
321 if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj))
322 return -EBUSY;
323
324 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
325 drm_dbg(obj->base.dev,
326 "Attempting to obtain a purgeable object\n");
327 return -EFAULT;
328 }
329
330 return i915_gem_object_shmem_to_phys(obj);
331 }
332
333 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
334 #include "selftests/i915_gem_phys.c"
335 #endif
336