14cd92098Szrj /*
24cd92098Szrj * Copyright 2013 Advanced Micro Devices, Inc.
34cd92098Szrj *
44cd92098Szrj * Permission is hereby granted, free of charge, to any person obtaining a
54cd92098Szrj * copy of this software and associated documentation files (the "Software"),
64cd92098Szrj * to deal in the Software without restriction, including without limitation
74cd92098Szrj * the rights to use, copy, modify, merge, publish, distribute, sublicense,
84cd92098Szrj * and/or sell copies of the Software, and to permit persons to whom the
94cd92098Szrj * Software is furnished to do so, subject to the following conditions:
104cd92098Szrj *
114cd92098Szrj * The above copyright notice and this permission notice shall be included in
124cd92098Szrj * all copies or substantial portions of the Software.
134cd92098Szrj *
144cd92098Szrj * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
154cd92098Szrj * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
164cd92098Szrj * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
174cd92098Szrj * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
184cd92098Szrj * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
194cd92098Szrj * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
204cd92098Szrj * OTHER DEALINGS IN THE SOFTWARE.
214cd92098Szrj *
224cd92098Szrj * Authors: Alex Deucher
234cd92098Szrj */
244cd92098Szrj #include <drm/drmP.h>
254cd92098Szrj #include "radeon.h"
264cd92098Szrj #include "radeon_asic.h"
27c6f73aabSFrançois Tigeot #ifdef TRACE_TODO
28c6f73aabSFrançois Tigeot #include "radeon_trace.h"
29c6f73aabSFrançois Tigeot #endif
304cd92098Szrj #include "sid.h"
314cd92098Szrj
324cd92098Szrj /**
334cd92098Szrj * si_dma_is_lockup - Check if the DMA engine is locked up
344cd92098Szrj *
354cd92098Szrj * @rdev: radeon_device pointer
364cd92098Szrj * @ring: radeon_ring structure holding ring information
374cd92098Szrj *
384cd92098Szrj * Check if the async DMA engine is locked up.
394cd92098Szrj * Returns true if the engine appears to be locked up, false if not.
404cd92098Szrj */
si_dma_is_lockup(struct radeon_device * rdev,struct radeon_ring * ring)414cd92098Szrj bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
424cd92098Szrj {
434cd92098Szrj u32 reset_mask = si_gpu_check_soft_reset(rdev);
444cd92098Szrj u32 mask;
454cd92098Szrj
464cd92098Szrj if (ring->idx == R600_RING_TYPE_DMA_INDEX)
474cd92098Szrj mask = RADEON_RESET_DMA;
484cd92098Szrj else
494cd92098Szrj mask = RADEON_RESET_DMA1;
504cd92098Szrj
514cd92098Szrj if (!(reset_mask & mask)) {
52c6f73aabSFrançois Tigeot radeon_ring_lockup_update(rdev, ring);
534cd92098Szrj return false;
544cd92098Szrj }
554cd92098Szrj return radeon_ring_test_lockup(rdev, ring);
564cd92098Szrj }
574cd92098Szrj
584cd92098Szrj /**
59c6f73aabSFrançois Tigeot * si_dma_vm_copy_pages - update PTEs by copying them from the GART
60c6f73aabSFrançois Tigeot *
61c6f73aabSFrançois Tigeot * @rdev: radeon_device pointer
62c6f73aabSFrançois Tigeot * @ib: indirect buffer to fill with commands
63c6f73aabSFrançois Tigeot * @pe: addr of the page entry
64c6f73aabSFrançois Tigeot * @src: src addr where to copy from
65c6f73aabSFrançois Tigeot * @count: number of page entries to update
66c6f73aabSFrançois Tigeot *
67c6f73aabSFrançois Tigeot * Update PTEs by copying them from the GART using the DMA (SI).
68c6f73aabSFrançois Tigeot */
si_dma_vm_copy_pages(struct radeon_device * rdev,struct radeon_ib * ib,uint64_t pe,uint64_t src,unsigned count)69c6f73aabSFrançois Tigeot void si_dma_vm_copy_pages(struct radeon_device *rdev,
70c6f73aabSFrançois Tigeot struct radeon_ib *ib,
71c6f73aabSFrançois Tigeot uint64_t pe, uint64_t src,
72c6f73aabSFrançois Tigeot unsigned count)
73c6f73aabSFrançois Tigeot {
74c6f73aabSFrançois Tigeot while (count) {
75c6f73aabSFrançois Tigeot unsigned bytes = count * 8;
76c6f73aabSFrançois Tigeot if (bytes > 0xFFFF8)
77c6f73aabSFrançois Tigeot bytes = 0xFFFF8;
78c6f73aabSFrançois Tigeot
79c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_COPY,
80c6f73aabSFrançois Tigeot 1, 0, 0, bytes);
81c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = lower_32_bits(pe);
82c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = lower_32_bits(src);
83c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
84c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = upper_32_bits(src) & 0xff;
85c6f73aabSFrançois Tigeot
86c6f73aabSFrançois Tigeot pe += bytes;
87c6f73aabSFrançois Tigeot src += bytes;
88c6f73aabSFrançois Tigeot count -= bytes / 8;
89c6f73aabSFrançois Tigeot }
90c6f73aabSFrançois Tigeot }
91c6f73aabSFrançois Tigeot
92c6f73aabSFrançois Tigeot /**
93c6f73aabSFrançois Tigeot * si_dma_vm_write_pages - update PTEs by writing them manually
94c6f73aabSFrançois Tigeot *
95c6f73aabSFrançois Tigeot * @rdev: radeon_device pointer
96c6f73aabSFrançois Tigeot * @ib: indirect buffer to fill with commands
97c6f73aabSFrançois Tigeot * @pe: addr of the page entry
98c6f73aabSFrançois Tigeot * @addr: dst addr to write into pe
99c6f73aabSFrançois Tigeot * @count: number of page entries to update
100c6f73aabSFrançois Tigeot * @incr: increase next addr by incr bytes
101c6f73aabSFrançois Tigeot * @flags: access flags
102c6f73aabSFrançois Tigeot *
103c6f73aabSFrançois Tigeot * Update PTEs by writing them manually using the DMA (SI).
104c6f73aabSFrançois Tigeot */
si_dma_vm_write_pages(struct radeon_device * rdev,struct radeon_ib * ib,uint64_t pe,uint64_t addr,unsigned count,uint32_t incr,uint32_t flags)105c6f73aabSFrançois Tigeot void si_dma_vm_write_pages(struct radeon_device *rdev,
106c6f73aabSFrançois Tigeot struct radeon_ib *ib,
107c6f73aabSFrançois Tigeot uint64_t pe,
108c6f73aabSFrançois Tigeot uint64_t addr, unsigned count,
109c6f73aabSFrançois Tigeot uint32_t incr, uint32_t flags)
110c6f73aabSFrançois Tigeot {
111c6f73aabSFrançois Tigeot uint64_t value;
112c6f73aabSFrançois Tigeot unsigned ndw;
113c6f73aabSFrançois Tigeot
114c6f73aabSFrançois Tigeot while (count) {
115c6f73aabSFrançois Tigeot ndw = count * 2;
116c6f73aabSFrançois Tigeot if (ndw > 0xFFFFE)
117c6f73aabSFrançois Tigeot ndw = 0xFFFFE;
118c6f73aabSFrançois Tigeot
119c6f73aabSFrançois Tigeot /* for non-physically contiguous pages (system) */
120c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
121c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = pe;
122c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
123c6f73aabSFrançois Tigeot for (; ndw > 0; ndw -= 2, --count, pe += 8) {
124c6f73aabSFrançois Tigeot if (flags & R600_PTE_SYSTEM) {
125c6f73aabSFrançois Tigeot value = radeon_vm_map_gart(rdev, addr);
126c6f73aabSFrançois Tigeot } else if (flags & R600_PTE_VALID) {
127c6f73aabSFrançois Tigeot value = addr;
128c6f73aabSFrançois Tigeot } else {
129c6f73aabSFrançois Tigeot value = 0;
130c6f73aabSFrançois Tigeot }
131c6f73aabSFrançois Tigeot addr += incr;
132c6f73aabSFrançois Tigeot value |= flags;
133c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = value;
134c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = upper_32_bits(value);
135c6f73aabSFrançois Tigeot }
136c6f73aabSFrançois Tigeot }
137c6f73aabSFrançois Tigeot }
138c6f73aabSFrançois Tigeot
139c6f73aabSFrançois Tigeot /**
140c6f73aabSFrançois Tigeot * si_dma_vm_set_pages - update the page tables using the DMA
1414cd92098Szrj *
1424cd92098Szrj * @rdev: radeon_device pointer
1434cd92098Szrj * @ib: indirect buffer to fill with commands
1444cd92098Szrj * @pe: addr of the page entry
1454cd92098Szrj * @addr: dst addr to write into pe
1464cd92098Szrj * @count: number of page entries to update
1474cd92098Szrj * @incr: increase next addr by incr bytes
1484cd92098Szrj * @flags: access flags
1494cd92098Szrj *
1504cd92098Szrj * Update the page tables using the DMA (SI).
1514cd92098Szrj */
si_dma_vm_set_pages(struct radeon_device * rdev,struct radeon_ib * ib,uint64_t pe,uint64_t addr,unsigned count,uint32_t incr,uint32_t flags)152c6f73aabSFrançois Tigeot void si_dma_vm_set_pages(struct radeon_device *rdev,
1534cd92098Szrj struct radeon_ib *ib,
1544cd92098Szrj uint64_t pe,
1554cd92098Szrj uint64_t addr, unsigned count,
1564cd92098Szrj uint32_t incr, uint32_t flags)
1574cd92098Szrj {
1584cd92098Szrj uint64_t value;
1594cd92098Szrj unsigned ndw;
1604cd92098Szrj
1614cd92098Szrj while (count) {
1624cd92098Szrj ndw = count * 2;
1634cd92098Szrj if (ndw > 0xFFFFE)
1644cd92098Szrj ndw = 0xFFFFE;
1654cd92098Szrj
166c6f73aabSFrançois Tigeot if (flags & R600_PTE_VALID)
1674cd92098Szrj value = addr;
1684cd92098Szrj else
1694cd92098Szrj value = 0;
170c6f73aabSFrançois Tigeot
1714cd92098Szrj /* for physically contiguous pages (vram) */
1724cd92098Szrj ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
1734cd92098Szrj ib->ptr[ib->length_dw++] = pe; /* dst addr */
1744cd92098Szrj ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
175c6f73aabSFrançois Tigeot ib->ptr[ib->length_dw++] = flags; /* mask */
1764cd92098Szrj ib->ptr[ib->length_dw++] = 0;
1774cd92098Szrj ib->ptr[ib->length_dw++] = value; /* value */
1784cd92098Szrj ib->ptr[ib->length_dw++] = upper_32_bits(value);
1794cd92098Szrj ib->ptr[ib->length_dw++] = incr; /* increment size */
1804cd92098Szrj ib->ptr[ib->length_dw++] = 0;
1814cd92098Szrj pe += ndw * 4;
1824cd92098Szrj addr += (ndw / 2) * incr;
1834cd92098Szrj count -= ndw / 2;
1844cd92098Szrj }
1854cd92098Szrj }
1864cd92098Szrj
si_dma_vm_flush(struct radeon_device * rdev,struct radeon_ring * ring,unsigned vm_id,uint64_t pd_addr)187*7dcf36dcSFrançois Tigeot void si_dma_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
188*7dcf36dcSFrançois Tigeot unsigned vm_id, uint64_t pd_addr)
189*7dcf36dcSFrançois Tigeot
1904cd92098Szrj {
191ee479021SImre Vadász radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
192*7dcf36dcSFrançois Tigeot if (vm_id < 8) {
193*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2));
1944cd92098Szrj } else {
195*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2));
1964cd92098Szrj }
197*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, pd_addr >> 12);
1984cd92098Szrj
1994cd92098Szrj /* flush hdp cache */
2004cd92098Szrj radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
2014cd92098Szrj radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
2024cd92098Szrj radeon_ring_write(ring, 1);
2034cd92098Szrj
2044cd92098Szrj /* bits 0-7 are the VM contexts0-7 */
2054cd92098Szrj radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
2064cd92098Szrj radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
207*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, 1 << vm_id);
208*7dcf36dcSFrançois Tigeot
209*7dcf36dcSFrançois Tigeot /* wait for invalidate to complete */
210*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_POLL_REG_MEM, 0, 0, 0, 0));
211*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, VM_INVALIDATE_REQUEST);
212*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, 0xff << 16); /* retry */
213*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, 1 << vm_id); /* mask */
214*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, 0); /* value */
215*7dcf36dcSFrançois Tigeot radeon_ring_write(ring, (0 << 28) | 0x20); /* func(always) | poll interval */
2164cd92098Szrj }
2174cd92098Szrj
2184cd92098Szrj /**
2194cd92098Szrj * si_copy_dma - copy pages using the DMA engine
2204cd92098Szrj *
2214cd92098Szrj * @rdev: radeon_device pointer
2224cd92098Szrj * @src_offset: src GPU address
2234cd92098Szrj * @dst_offset: dst GPU address
2244cd92098Szrj * @num_gpu_pages: number of GPU pages to xfer
2251cfef1a5SFrançois Tigeot * @resv: reservation object to sync to
2264cd92098Szrj *
2274cd92098Szrj * Copy GPU paging using the DMA engine (SI).
2284cd92098Szrj * Used by the radeon ttm implementation to move pages if
2294cd92098Szrj * registered as the asic copy callback.
2304cd92098Szrj */
si_copy_dma(struct radeon_device * rdev,uint64_t src_offset,uint64_t dst_offset,unsigned num_gpu_pages,struct reservation_object * resv)2311cfef1a5SFrançois Tigeot struct radeon_fence *si_copy_dma(struct radeon_device *rdev,
2324cd92098Szrj uint64_t src_offset, uint64_t dst_offset,
2334cd92098Szrj unsigned num_gpu_pages,
2341cfef1a5SFrançois Tigeot struct reservation_object *resv)
2354cd92098Szrj {
2361cfef1a5SFrançois Tigeot struct radeon_fence *fence;
237*7dcf36dcSFrançois Tigeot struct radeon_sync sync;
2384cd92098Szrj int ring_index = rdev->asic->copy.dma_ring_index;
2394cd92098Szrj struct radeon_ring *ring = &rdev->ring[ring_index];
2404cd92098Szrj u32 size_in_bytes, cur_size_in_bytes;
2414cd92098Szrj int i, num_loops;
2424cd92098Szrj int r = 0;
2434cd92098Szrj
244*7dcf36dcSFrançois Tigeot radeon_sync_create(&sync);
2454cd92098Szrj
2464cd92098Szrj size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
2474cd92098Szrj num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
2484cd92098Szrj r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
2494cd92098Szrj if (r) {
2504cd92098Szrj DRM_ERROR("radeon: moving bo (%d).\n", r);
251*7dcf36dcSFrançois Tigeot radeon_sync_free(rdev, &sync, NULL);
2521cfef1a5SFrançois Tigeot return ERR_PTR(r);
2534cd92098Szrj }
2544cd92098Szrj
255*7dcf36dcSFrançois Tigeot radeon_sync_resv(rdev, &sync, resv, false);
256*7dcf36dcSFrançois Tigeot radeon_sync_rings(rdev, &sync, ring->idx);
2574cd92098Szrj
2584cd92098Szrj for (i = 0; i < num_loops; i++) {
2594cd92098Szrj cur_size_in_bytes = size_in_bytes;
2604cd92098Szrj if (cur_size_in_bytes > 0xFFFFF)
2614cd92098Szrj cur_size_in_bytes = 0xFFFFF;
2624cd92098Szrj size_in_bytes -= cur_size_in_bytes;
2634cd92098Szrj radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
264c6f73aabSFrançois Tigeot radeon_ring_write(ring, lower_32_bits(dst_offset));
265c6f73aabSFrançois Tigeot radeon_ring_write(ring, lower_32_bits(src_offset));
2664cd92098Szrj radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
2674cd92098Szrj radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
2684cd92098Szrj src_offset += cur_size_in_bytes;
2694cd92098Szrj dst_offset += cur_size_in_bytes;
2704cd92098Szrj }
2714cd92098Szrj
2721cfef1a5SFrançois Tigeot r = radeon_fence_emit(rdev, &fence, ring->idx);
2734cd92098Szrj if (r) {
2744cd92098Szrj radeon_ring_unlock_undo(rdev, ring);
275*7dcf36dcSFrançois Tigeot radeon_sync_free(rdev, &sync, NULL);
2761cfef1a5SFrançois Tigeot return ERR_PTR(r);
2774cd92098Szrj }
2784cd92098Szrj
279c6f73aabSFrançois Tigeot radeon_ring_unlock_commit(rdev, ring, false);
280*7dcf36dcSFrançois Tigeot radeon_sync_free(rdev, &sync, fence);
2814cd92098Szrj
2821cfef1a5SFrançois Tigeot return fence;
2834cd92098Szrj }
284