1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2014-2016 Intel Corporation 5 */ 6 7 #include <linux/highmem.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/swap.h> 10 11 #include <drm/drm_cache.h> 12 #include <drm/drm_legacy.h> /* for drm_dmamem_alloc() */ 13 14 #include "gt/intel_gt.h" 15 #include "i915_drv.h" 16 #include "i915_gem_object.h" 17 #include "i915_gem_region.h" 18 #include "i915_gem_tiling.h" 19 #include "i915_scatterlist.h" 20 21 static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) 22 { 23 #ifdef __linux__ 24 struct address_space *mapping = obj->base.filp->f_mapping; 25 #else 26 struct drm_dmamem *dmah; 27 int flags = 0; 28 #endif 29 struct drm_i915_private *i915 = to_i915(obj->base.dev); 30 struct scatterlist *sg; 31 struct sg_table *st; 32 dma_addr_t dma; 33 void *vaddr; 34 void *dst; 35 int i; 36 37 /* Contiguous chunk, with a single scatterlist element */ 38 if (overflows_type(obj->base.size, sg->length)) 39 return -E2BIG; 40 41 if (GEM_WARN_ON(i915_gem_object_needs_bit17_swizzle(obj))) 42 return -EINVAL; 43 44 /* 45 * Always aligning to the object size, allows a single allocation 46 * to handle all possible callers, and given typical object sizes, 47 * the alignment of the buddy allocation will naturally match. 48 */ 49 #ifdef __linux__ 50 vaddr = dma_alloc_coherent(obj->base.dev->dev, 51 roundup_pow_of_two(obj->base.size), 52 &dma, GFP_KERNEL); 53 if (!vaddr) 54 return -ENOMEM; 55 #else 56 dmah = drm_dmamem_alloc(i915->dmat, 57 roundup_pow_of_two(obj->base.size), 58 PAGE_SIZE, 1, 59 roundup_pow_of_two(obj->base.size), flags, 0); 60 if (dmah == NULL) 61 return -ENOMEM; 62 dma = dmah->map->dm_segs[0].ds_addr; 63 vaddr = dmah->kva; 64 #endif 65 66 st = kmalloc(sizeof(*st), GFP_KERNEL); 67 if (!st) 68 goto err_pci; 69 70 if (sg_alloc_table(st, 1, GFP_KERNEL)) 71 goto err_st; 72 73 sg = st->sgl; 74 sg->offset = 0; 75 sg->length = obj->base.size; 76 77 #ifdef __linux__ 78 sg_assign_page(sg, (struct page *)vaddr); 79 #else 80 sg_assign_page(sg, (struct vm_page *)dmah); 81 #endif 82 sg_dma_address(sg) = dma; 83 sg_dma_len(sg) = obj->base.size; 84 85 dst = vaddr; 86 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 87 struct vm_page *page; 88 void *src; 89 90 #ifdef __linux__ 91 page = shmem_read_mapping_page(mapping, i); 92 if (IS_ERR(page)) 93 goto err_st; 94 #else 95 struct pglist plist; 96 TAILQ_INIT(&plist); 97 if (uvm_obj_wire(obj->base.uao, i * PAGE_SIZE, 98 (i + 1) * PAGE_SIZE, &plist)) 99 goto err_st; 100 page = TAILQ_FIRST(&plist); 101 #endif 102 103 src = kmap_atomic(page); 104 memcpy(dst, src, PAGE_SIZE); 105 drm_clflush_virt_range(dst, PAGE_SIZE); 106 kunmap_atomic(src); 107 108 #ifdef __linux__ 109 put_page(page); 110 #else 111 uvm_obj_unwire(obj->base.uao, i * PAGE_SIZE, 112 (i + 1) * PAGE_SIZE); 113 #endif 114 dst += PAGE_SIZE; 115 } 116 117 intel_gt_chipset_flush(to_gt(i915)); 118 119 /* We're no longer struct page backed */ 120 obj->mem_flags &= ~I915_BO_FLAG_STRUCT_PAGE; 121 __i915_gem_object_set_pages(obj, st); 122 123 return 0; 124 125 err_st: 126 kfree(st); 127 err_pci: 128 #ifdef __linux__ 129 dma_free_coherent(obj->base.dev->dev, 130 roundup_pow_of_two(obj->base.size), 131 vaddr, dma); 132 #else 133 drm_dmamem_free(i915->dmat, dmah); 134 #endif 135 return -ENOMEM; 136 } 137 138 void 139 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, 140 struct sg_table *pages) 141 { 142 dma_addr_t dma = sg_dma_address(pages->sgl); 143 #ifdef __linux__ 144 void *vaddr = sg_page(pages->sgl); 145 #else 146 struct drm_dmamem *dmah = (void *)sg_page(pages->sgl); 147 void *vaddr = dmah->kva; 148 struct drm_i915_private *i915 = to_i915(obj->base.dev); 149 #endif 150 151 __i915_gem_object_release_shmem(obj, pages, false); 152 153 if (obj->mm.dirty) { 154 #ifdef __linux__ 155 struct address_space *mapping = obj->base.filp->f_mapping; 156 #endif 157 void *src = vaddr; 158 int i; 159 160 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) { 161 struct vm_page *page; 162 char *dst; 163 164 #ifdef __linux__ 165 page = shmem_read_mapping_page(mapping, i); 166 if (IS_ERR(page)) 167 continue; 168 #else 169 struct pglist plist; 170 TAILQ_INIT(&plist); 171 if (uvm_obj_wire(obj->base.uao, i * PAGE_SIZE, 172 (i + 1) * PAGE_SIZE, &plist)) 173 continue; 174 page = TAILQ_FIRST(&plist); 175 #endif 176 177 dst = kmap_atomic(page); 178 drm_clflush_virt_range(src, PAGE_SIZE); 179 memcpy(dst, src, PAGE_SIZE); 180 kunmap_atomic(dst); 181 182 set_page_dirty(page); 183 #ifdef __linux__ 184 if (obj->mm.madv == I915_MADV_WILLNEED) 185 mark_page_accessed(page); 186 put_page(page); 187 #else 188 uvm_obj_unwire(obj->base.uao, i * PAGE_SIZE, 189 (i + 1) * PAGE_SIZE); 190 #endif 191 192 src += PAGE_SIZE; 193 } 194 obj->mm.dirty = false; 195 } 196 197 sg_free_table(pages); 198 kfree(pages); 199 200 #ifdef __linux__ 201 dma_free_coherent(obj->base.dev->dev, 202 roundup_pow_of_two(obj->base.size), 203 vaddr, dma); 204 #else 205 drm_dmamem_free(i915->dmat, dmah); 206 #endif 207 } 208 209 int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj, 210 const struct drm_i915_gem_pwrite *args) 211 { 212 #ifdef __linux__ 213 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset; 214 #else 215 struct drm_dmamem *dmah = (void *)sg_page(obj->mm.pages->sgl); 216 void *vaddr = dmah->kva + args->offset; 217 #endif 218 char __user *user_data = u64_to_user_ptr(args->data_ptr); 219 struct drm_i915_private *i915 = to_i915(obj->base.dev); 220 int err; 221 222 err = i915_gem_object_wait(obj, 223 I915_WAIT_INTERRUPTIBLE | 224 I915_WAIT_ALL, 225 MAX_SCHEDULE_TIMEOUT); 226 if (err) 227 return err; 228 229 /* 230 * We manually control the domain here and pretend that it 231 * remains coherent i.e. in the GTT domain, like shmem_pwrite. 232 */ 233 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU); 234 235 if (copy_from_user(vaddr, user_data, args->size)) 236 return -EFAULT; 237 238 drm_clflush_virt_range(vaddr, args->size); 239 intel_gt_chipset_flush(to_gt(i915)); 240 241 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU); 242 return 0; 243 } 244 245 int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj, 246 const struct drm_i915_gem_pread *args) 247 { 248 #ifdef __linux__ 249 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset; 250 #else 251 struct drm_dmamem *dmah = (void *)sg_page(obj->mm.pages->sgl); 252 void *vaddr = dmah->kva + args->offset; 253 #endif 254 char __user *user_data = u64_to_user_ptr(args->data_ptr); 255 int err; 256 257 err = i915_gem_object_wait(obj, 258 I915_WAIT_INTERRUPTIBLE, 259 MAX_SCHEDULE_TIMEOUT); 260 if (err) 261 return err; 262 263 drm_clflush_virt_range(vaddr, args->size); 264 if (copy_to_user(user_data, vaddr, args->size)) 265 return -EFAULT; 266 267 return 0; 268 } 269 270 static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj) 271 { 272 struct sg_table *pages; 273 int err; 274 275 pages = __i915_gem_object_unset_pages(obj); 276 277 err = i915_gem_object_get_pages_phys(obj); 278 if (err) 279 goto err_xfer; 280 281 /* Perma-pin (until release) the physical set of pages */ 282 __i915_gem_object_pin_pages(obj); 283 284 if (!IS_ERR_OR_NULL(pages)) 285 i915_gem_object_put_pages_shmem(obj, pages); 286 287 i915_gem_object_release_memory_region(obj); 288 return 0; 289 290 err_xfer: 291 if (!IS_ERR_OR_NULL(pages)) 292 __i915_gem_object_set_pages(obj, pages); 293 return err; 294 } 295 296 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) 297 { 298 int err; 299 300 assert_object_held(obj); 301 302 if (align > obj->base.size) 303 return -EINVAL; 304 305 if (!i915_gem_object_is_shmem(obj)) 306 return -EINVAL; 307 308 if (!i915_gem_object_has_struct_page(obj)) 309 return 0; 310 311 err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE); 312 if (err) 313 return err; 314 315 if (obj->mm.madv != I915_MADV_WILLNEED) 316 return -EFAULT; 317 318 if (i915_gem_object_has_tiling_quirk(obj)) 319 return -EFAULT; 320 321 if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj)) 322 return -EBUSY; 323 324 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) { 325 drm_dbg(obj->base.dev, 326 "Attempting to obtain a purgeable object\n"); 327 return -EFAULT; 328 } 329 330 return i915_gem_object_shmem_to_phys(obj); 331 } 332 333 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 334 #include "selftests/i915_gem_phys.c" 335 #endif 336