1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28
29 #include <linux/pci.h>
30 #include <linux/vmalloc.h>
31
32 #include <uvm/uvm_glue.h>
33
34 #include <drm/radeon_drm.h>
35 #ifdef CONFIG_X86
36 #include <asm/set_memory.h>
37 #endif
38 #include "radeon.h"
39
40 /*
41 * GART
42 * The GART (Graphics Aperture Remapping Table) is an aperture
43 * in the GPU's address space. System pages can be mapped into
44 * the aperture and look like contiguous pages from the GPU's
45 * perspective. A page table maps the pages in the aperture
46 * to the actual backing pages in system memory.
47 *
48 * Radeon GPUs support both an internal GART, as described above,
49 * and AGP. AGP works similarly, but the GART table is configured
50 * and maintained by the northbridge rather than the driver.
51 * Radeon hw has a separate AGP aperture that is programmed to
52 * point to the AGP aperture provided by the northbridge and the
53 * requests are passed through to the northbridge aperture.
54 * Both AGP and internal GART can be used at the same time, however
55 * that is not currently supported by the driver.
56 *
57 * This file handles the common internal GART management.
58 */
59
60 /*
61 * Common GART table functions.
62 */
63 /**
64 * radeon_gart_table_ram_alloc - allocate system ram for gart page table
65 *
66 * @rdev: radeon_device pointer
67 *
68 * Allocate system memory for GART page table
69 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
70 * gart table to be in system memory.
71 * Returns 0 for success, -ENOMEM for failure.
72 */
73 #ifdef __linux__
radeon_gart_table_ram_alloc(struct radeon_device * rdev)74 int radeon_gart_table_ram_alloc(struct radeon_device *rdev)
75 {
76 void *ptr;
77
78 ptr = dma_alloc_coherent(&rdev->pdev->dev, rdev->gart.table_size,
79 &rdev->gart.table_addr, GFP_KERNEL);
80 if (!ptr)
81 return -ENOMEM;
82
83 #ifdef CONFIG_X86
84 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
85 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
86 set_memory_uc((unsigned long)ptr,
87 rdev->gart.table_size >> PAGE_SHIFT);
88 }
89 #endif
90 rdev->gart.ptr = ptr;
91 return 0;
92 }
93 #else
radeon_gart_table_ram_alloc(struct radeon_device * rdev)94 int radeon_gart_table_ram_alloc(struct radeon_device *rdev)
95 {
96 struct drm_dmamem *dmah;
97 int flags = 0;
98
99 #ifdef CONFIG_X86
100 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
101 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
102 flags |= BUS_DMA_NOCACHE;
103 }
104 #endif
105 dmah = drm_dmamem_alloc(rdev->dmat, rdev->gart.table_size,
106 rdev->gart.table_size, 1, rdev->gart.table_size, flags, 0);
107 if (dmah == NULL) {
108 return -ENOMEM;
109 }
110 rdev->gart.dmah = dmah;
111 rdev->gart.table_addr = dmah->map->dm_segs[0].ds_addr;
112 rdev->gart.ptr = dmah->kva;
113 memset((void *)rdev->gart.ptr, 0, rdev->gart.table_size);
114 return 0;
115 }
116 #endif
117
118 /**
119 * radeon_gart_table_ram_free - free system ram for gart page table
120 *
121 * @rdev: radeon_device pointer
122 *
123 * Free system memory for GART page table
124 * (r1xx-r3xx, non-pcie r4xx, rs400). These asics require the
125 * gart table to be in system memory.
126 */
127 #ifdef __linux__
radeon_gart_table_ram_free(struct radeon_device * rdev)128 void radeon_gart_table_ram_free(struct radeon_device *rdev)
129 {
130 if (!rdev->gart.ptr)
131 return;
132
133 #ifdef CONFIG_X86
134 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
135 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
136 set_memory_wb((unsigned long)rdev->gart.ptr,
137 rdev->gart.table_size >> PAGE_SHIFT);
138 }
139 #endif
140 dma_free_coherent(&rdev->pdev->dev, rdev->gart.table_size,
141 (void *)rdev->gart.ptr, rdev->gart.table_addr);
142 rdev->gart.ptr = NULL;
143 rdev->gart.table_addr = 0;
144 }
145 #else
radeon_gart_table_ram_free(struct radeon_device * rdev)146 void radeon_gart_table_ram_free(struct radeon_device *rdev)
147 {
148 if (rdev->gart.ptr == NULL) {
149 return;
150 }
151 #if defined (CONFIG_X86) && defined(__linux__)
152 if (rdev->family == CHIP_RS400 || rdev->family == CHIP_RS480 ||
153 rdev->family == CHIP_RS690 || rdev->family == CHIP_RS740) {
154 set_memory_wb((unsigned long)rdev->gart.ptr,
155 rdev->gart.table_size >> PAGE_SHIFT);
156 }
157 #endif
158 drm_dmamem_free(rdev->dmat, rdev->gart.dmah);
159 rdev->gart.ptr = NULL;
160 rdev->gart.table_addr = 0;
161 }
162 #endif
163
164 /**
165 * radeon_gart_table_vram_alloc - allocate vram for gart page table
166 *
167 * @rdev: radeon_device pointer
168 *
169 * Allocate video memory for GART page table
170 * (pcie r4xx, r5xx+). These asics require the
171 * gart table to be in video memory.
172 * Returns 0 for success, error for failure.
173 */
radeon_gart_table_vram_alloc(struct radeon_device * rdev)174 int radeon_gart_table_vram_alloc(struct radeon_device *rdev)
175 {
176 int r;
177
178 if (rdev->gart.robj == NULL) {
179 r = radeon_bo_create(rdev, rdev->gart.table_size,
180 PAGE_SIZE, true, RADEON_GEM_DOMAIN_VRAM,
181 0, NULL, NULL, &rdev->gart.robj);
182 if (r)
183 return r;
184 }
185 return 0;
186 }
187
188 /**
189 * radeon_gart_table_vram_pin - pin gart page table in vram
190 *
191 * @rdev: radeon_device pointer
192 *
193 * Pin the GART page table in vram so it will not be moved
194 * by the memory manager (pcie r4xx, r5xx+). These asics require the
195 * gart table to be in video memory.
196 * Returns 0 for success, error for failure.
197 */
radeon_gart_table_vram_pin(struct radeon_device * rdev)198 int radeon_gart_table_vram_pin(struct radeon_device *rdev)
199 {
200 uint64_t gpu_addr;
201 int r;
202
203 r = radeon_bo_reserve(rdev->gart.robj, false);
204 if (unlikely(r != 0))
205 return r;
206 r = radeon_bo_pin(rdev->gart.robj,
207 RADEON_GEM_DOMAIN_VRAM, &gpu_addr);
208 if (r) {
209 radeon_bo_unreserve(rdev->gart.robj);
210 return r;
211 }
212 r = radeon_bo_kmap(rdev->gart.robj, &rdev->gart.ptr);
213 if (r)
214 radeon_bo_unpin(rdev->gart.robj);
215 radeon_bo_unreserve(rdev->gart.robj);
216 rdev->gart.table_addr = gpu_addr;
217
218 if (!r) {
219 int i;
220
221 /* We might have dropped some GART table updates while it wasn't
222 * mapped, restore all entries
223 */
224 for (i = 0; i < rdev->gart.num_gpu_pages; i++)
225 radeon_gart_set_page(rdev, i, rdev->gart.pages_entry[i]);
226 mb();
227 radeon_gart_tlb_flush(rdev);
228 }
229
230 return r;
231 }
232
233 /**
234 * radeon_gart_table_vram_unpin - unpin gart page table in vram
235 *
236 * @rdev: radeon_device pointer
237 *
238 * Unpin the GART page table in vram (pcie r4xx, r5xx+).
239 * These asics require the gart table to be in video memory.
240 */
radeon_gart_table_vram_unpin(struct radeon_device * rdev)241 void radeon_gart_table_vram_unpin(struct radeon_device *rdev)
242 {
243 int r;
244
245 if (!rdev->gart.robj)
246 return;
247
248 r = radeon_bo_reserve(rdev->gart.robj, false);
249 if (likely(r == 0)) {
250 radeon_bo_kunmap(rdev->gart.robj);
251 radeon_bo_unpin(rdev->gart.robj);
252 radeon_bo_unreserve(rdev->gart.robj);
253 rdev->gart.ptr = NULL;
254 }
255 }
256
257 /**
258 * radeon_gart_table_vram_free - free gart page table vram
259 *
260 * @rdev: radeon_device pointer
261 *
262 * Free the video memory used for the GART page table
263 * (pcie r4xx, r5xx+). These asics require the gart table to
264 * be in video memory.
265 */
radeon_gart_table_vram_free(struct radeon_device * rdev)266 void radeon_gart_table_vram_free(struct radeon_device *rdev)
267 {
268 if (!rdev->gart.robj)
269 return;
270
271 radeon_bo_unref(&rdev->gart.robj);
272 }
273
274 /*
275 * Common gart functions.
276 */
277 /**
278 * radeon_gart_unbind - unbind pages from the gart page table
279 *
280 * @rdev: radeon_device pointer
281 * @offset: offset into the GPU's gart aperture
282 * @pages: number of pages to unbind
283 *
284 * Unbinds the requested pages from the gart page table and
285 * replaces them with the dummy page (all asics).
286 */
radeon_gart_unbind(struct radeon_device * rdev,unsigned int offset,int pages)287 void radeon_gart_unbind(struct radeon_device *rdev, unsigned int offset,
288 int pages)
289 {
290 unsigned int t, p;
291 int i, j;
292
293 if (!rdev->gart.ready) {
294 WARN(1, "trying to unbind memory from uninitialized GART !\n");
295 return;
296 }
297 t = offset / RADEON_GPU_PAGE_SIZE;
298 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
299 for (i = 0; i < pages; i++, p++) {
300 if (rdev->gart.pages[p]) {
301 rdev->gart.pages[p] = NULL;
302 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
303 rdev->gart.pages_entry[t] = rdev->dummy_page.entry;
304 if (rdev->gart.ptr) {
305 radeon_gart_set_page(rdev, t,
306 rdev->dummy_page.entry);
307 }
308 }
309 }
310 }
311 if (rdev->gart.ptr) {
312 mb();
313 radeon_gart_tlb_flush(rdev);
314 }
315 }
316
317 /**
318 * radeon_gart_bind - bind pages into the gart page table
319 *
320 * @rdev: radeon_device pointer
321 * @offset: offset into the GPU's gart aperture
322 * @pages: number of pages to bind
323 * @pagelist: pages to bind
324 * @dma_addr: DMA addresses of pages
325 * @flags: RADEON_GART_PAGE_* flags
326 *
327 * Binds the requested pages to the gart page table
328 * (all asics).
329 * Returns 0 for success, -EINVAL for failure.
330 */
radeon_gart_bind(struct radeon_device * rdev,unsigned int offset,int pages,struct vm_page ** pagelist,dma_addr_t * dma_addr,uint32_t flags)331 int radeon_gart_bind(struct radeon_device *rdev, unsigned int offset,
332 int pages, struct vm_page **pagelist, dma_addr_t *dma_addr,
333 uint32_t flags)
334 {
335 unsigned int t, p;
336 uint64_t page_base, page_entry;
337 int i, j;
338
339 if (!rdev->gart.ready) {
340 WARN(1, "trying to bind memory to uninitialized GART !\n");
341 return -EINVAL;
342 }
343 t = offset / RADEON_GPU_PAGE_SIZE;
344 p = t / (PAGE_SIZE / RADEON_GPU_PAGE_SIZE);
345
346 for (i = 0; i < pages; i++, p++) {
347 rdev->gart.pages[p] = pagelist ? pagelist[i] :
348 uvm_atopg(rdev->dummy_page.addr);
349 page_base = dma_addr[i];
350 for (j = 0; j < (PAGE_SIZE / RADEON_GPU_PAGE_SIZE); j++, t++) {
351 page_entry = radeon_gart_get_page_entry(page_base, flags);
352 rdev->gart.pages_entry[t] = page_entry;
353 if (rdev->gart.ptr)
354 radeon_gart_set_page(rdev, t, page_entry);
355
356 page_base += RADEON_GPU_PAGE_SIZE;
357 }
358 }
359 if (rdev->gart.ptr) {
360 mb();
361 radeon_gart_tlb_flush(rdev);
362 }
363 return 0;
364 }
365
366 /**
367 * radeon_gart_init - init the driver info for managing the gart
368 *
369 * @rdev: radeon_device pointer
370 *
371 * Allocate the dummy page and init the gart driver info (all asics).
372 * Returns 0 for success, error for failure.
373 */
radeon_gart_init(struct radeon_device * rdev)374 int radeon_gart_init(struct radeon_device *rdev)
375 {
376 int r, i;
377
378 if (rdev->gart.pages)
379 return 0;
380
381 /* We need PAGE_SIZE >= RADEON_GPU_PAGE_SIZE */
382 if (PAGE_SIZE < RADEON_GPU_PAGE_SIZE) {
383 DRM_ERROR("Page size is smaller than GPU page size!\n");
384 return -EINVAL;
385 }
386 r = radeon_dummy_page_init(rdev);
387 if (r)
388 return r;
389 /* Compute table size */
390 rdev->gart.num_cpu_pages = rdev->mc.gtt_size / PAGE_SIZE;
391 rdev->gart.num_gpu_pages = rdev->mc.gtt_size / RADEON_GPU_PAGE_SIZE;
392 DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n",
393 rdev->gart.num_cpu_pages, rdev->gart.num_gpu_pages);
394 /* Allocate pages table */
395 rdev->gart.pages = vzalloc(array_size(sizeof(void *),
396 rdev->gart.num_cpu_pages));
397 if (rdev->gart.pages == NULL) {
398 radeon_gart_fini(rdev);
399 return -ENOMEM;
400 }
401 rdev->gart.pages_entry = vmalloc(array_size(sizeof(uint64_t),
402 rdev->gart.num_gpu_pages));
403 if (rdev->gart.pages_entry == NULL) {
404 radeon_gart_fini(rdev);
405 return -ENOMEM;
406 }
407 /* set GART entry to point to the dummy page by default */
408 for (i = 0; i < rdev->gart.num_gpu_pages; i++)
409 rdev->gart.pages_entry[i] = rdev->dummy_page.entry;
410 return 0;
411 }
412
413 /**
414 * radeon_gart_fini - tear down the driver info for managing the gart
415 *
416 * @rdev: radeon_device pointer
417 *
418 * Tear down the gart driver info and free the dummy page (all asics).
419 */
radeon_gart_fini(struct radeon_device * rdev)420 void radeon_gart_fini(struct radeon_device *rdev)
421 {
422 if (rdev->gart.ready) {
423 /* unbind pages */
424 radeon_gart_unbind(rdev, 0, rdev->gart.num_cpu_pages);
425 }
426 rdev->gart.ready = false;
427 vfree(rdev->gart.pages);
428 vfree(rdev->gart.pages_entry);
429 rdev->gart.pages = NULL;
430 rdev->gart.pages_entry = NULL;
431
432 radeon_dummy_page_fini(rdev);
433 }
434