1b843c749SSergey Zigachev /*
2b843c749SSergey Zigachev * Copyright 2016 Advanced Micro Devices, Inc.
3b843c749SSergey Zigachev * All Rights Reserved.
4b843c749SSergey Zigachev *
5b843c749SSergey Zigachev * Permission is hereby granted, free of charge, to any person obtaining a
6b843c749SSergey Zigachev * copy of this software and associated documentation files (the
7b843c749SSergey Zigachev * "Software"), to deal in the Software without restriction, including
8b843c749SSergey Zigachev * without limitation the rights to use, copy, modify, merge, publish,
9b843c749SSergey Zigachev * distribute, sub license, and/or sell copies of the Software, and to
10b843c749SSergey Zigachev * permit persons to whom the Software is furnished to do so, subject to
11b843c749SSergey Zigachev * the following conditions:
12b843c749SSergey Zigachev *
13b843c749SSergey Zigachev * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14b843c749SSergey Zigachev * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15b843c749SSergey Zigachev * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16b843c749SSergey Zigachev * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17b843c749SSergey Zigachev * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18b843c749SSergey Zigachev * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19b843c749SSergey Zigachev * USE OR OTHER DEALINGS IN THE SOFTWARE.
20b843c749SSergey Zigachev *
21b843c749SSergey Zigachev * The above copyright notice and this permission notice (including the
22b843c749SSergey Zigachev * next paragraph) shall be included in all copies or substantial portions
23b843c749SSergey Zigachev * of the Software.
24b843c749SSergey Zigachev *
25b843c749SSergey Zigachev */
26b843c749SSergey Zigachev
27b843c749SSergey Zigachev #include <linux/firmware.h>
28b843c749SSergey Zigachev #include <drm/drmP.h>
29b843c749SSergey Zigachev #include "amdgpu.h"
30b843c749SSergey Zigachev #include "amdgpu_vce.h"
31b843c749SSergey Zigachev #include "soc15.h"
32b843c749SSergey Zigachev #include "soc15d.h"
33b843c749SSergey Zigachev #include "soc15_common.h"
34b843c749SSergey Zigachev #include "mmsch_v1_0.h"
35b843c749SSergey Zigachev
36b843c749SSergey Zigachev #include "vce/vce_4_0_offset.h"
37b843c749SSergey Zigachev #include "vce/vce_4_0_default.h"
38b843c749SSergey Zigachev #include "vce/vce_4_0_sh_mask.h"
39b843c749SSergey Zigachev #include "mmhub/mmhub_1_0_offset.h"
40b843c749SSergey Zigachev #include "mmhub/mmhub_1_0_sh_mask.h"
41b843c749SSergey Zigachev
42b843c749SSergey Zigachev #include "ivsrcid/vce/irqsrcs_vce_4_0.h"
43b843c749SSergey Zigachev
44b843c749SSergey Zigachev #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK 0x02
45b843c749SSergey Zigachev
46b843c749SSergey Zigachev #define VCE_V4_0_FW_SIZE (384 * 1024)
47b843c749SSergey Zigachev #define VCE_V4_0_STACK_SIZE (64 * 1024)
48b843c749SSergey Zigachev #define VCE_V4_0_DATA_SIZE ((16 * 1024 * AMDGPU_MAX_VCE_HANDLES) + (52 * 1024))
49b843c749SSergey Zigachev
50b843c749SSergey Zigachev static void vce_v4_0_mc_resume(struct amdgpu_device *adev);
51b843c749SSergey Zigachev static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev);
52b843c749SSergey Zigachev static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev);
53b843c749SSergey Zigachev
54b843c749SSergey Zigachev /**
55b843c749SSergey Zigachev * vce_v4_0_ring_get_rptr - get read pointer
56b843c749SSergey Zigachev *
57b843c749SSergey Zigachev * @ring: amdgpu_ring pointer
58b843c749SSergey Zigachev *
59b843c749SSergey Zigachev * Returns the current hardware read pointer
60b843c749SSergey Zigachev */
vce_v4_0_ring_get_rptr(struct amdgpu_ring * ring)61b843c749SSergey Zigachev static uint64_t vce_v4_0_ring_get_rptr(struct amdgpu_ring *ring)
62b843c749SSergey Zigachev {
63b843c749SSergey Zigachev struct amdgpu_device *adev = ring->adev;
64b843c749SSergey Zigachev
65b843c749SSergey Zigachev if (ring->me == 0)
66b843c749SSergey Zigachev return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR));
67b843c749SSergey Zigachev else if (ring->me == 1)
68b843c749SSergey Zigachev return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR2));
69b843c749SSergey Zigachev else
70b843c749SSergey Zigachev return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR3));
71b843c749SSergey Zigachev }
72b843c749SSergey Zigachev
73b843c749SSergey Zigachev /**
74b843c749SSergey Zigachev * vce_v4_0_ring_get_wptr - get write pointer
75b843c749SSergey Zigachev *
76b843c749SSergey Zigachev * @ring: amdgpu_ring pointer
77b843c749SSergey Zigachev *
78b843c749SSergey Zigachev * Returns the current hardware write pointer
79b843c749SSergey Zigachev */
vce_v4_0_ring_get_wptr(struct amdgpu_ring * ring)80b843c749SSergey Zigachev static uint64_t vce_v4_0_ring_get_wptr(struct amdgpu_ring *ring)
81b843c749SSergey Zigachev {
82b843c749SSergey Zigachev struct amdgpu_device *adev = ring->adev;
83b843c749SSergey Zigachev
84b843c749SSergey Zigachev if (ring->use_doorbell)
85b843c749SSergey Zigachev return adev->wb.wb[ring->wptr_offs];
86b843c749SSergey Zigachev
87b843c749SSergey Zigachev if (ring->me == 0)
88b843c749SSergey Zigachev return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR));
89b843c749SSergey Zigachev else if (ring->me == 1)
90b843c749SSergey Zigachev return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2));
91b843c749SSergey Zigachev else
92b843c749SSergey Zigachev return RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3));
93b843c749SSergey Zigachev }
94b843c749SSergey Zigachev
95b843c749SSergey Zigachev /**
96b843c749SSergey Zigachev * vce_v4_0_ring_set_wptr - set write pointer
97b843c749SSergey Zigachev *
98b843c749SSergey Zigachev * @ring: amdgpu_ring pointer
99b843c749SSergey Zigachev *
100b843c749SSergey Zigachev * Commits the write pointer to the hardware
101b843c749SSergey Zigachev */
vce_v4_0_ring_set_wptr(struct amdgpu_ring * ring)102b843c749SSergey Zigachev static void vce_v4_0_ring_set_wptr(struct amdgpu_ring *ring)
103b843c749SSergey Zigachev {
104b843c749SSergey Zigachev struct amdgpu_device *adev = ring->adev;
105b843c749SSergey Zigachev
106b843c749SSergey Zigachev if (ring->use_doorbell) {
107b843c749SSergey Zigachev /* XXX check if swapping is necessary on BE */
108b843c749SSergey Zigachev adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
109b843c749SSergey Zigachev WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
110b843c749SSergey Zigachev return;
111b843c749SSergey Zigachev }
112b843c749SSergey Zigachev
113b843c749SSergey Zigachev if (ring->me == 0)
114b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR),
115b843c749SSergey Zigachev lower_32_bits(ring->wptr));
116b843c749SSergey Zigachev else if (ring->me == 1)
117b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2),
118b843c749SSergey Zigachev lower_32_bits(ring->wptr));
119b843c749SSergey Zigachev else
120b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3),
121b843c749SSergey Zigachev lower_32_bits(ring->wptr));
122b843c749SSergey Zigachev }
123b843c749SSergey Zigachev
vce_v4_0_firmware_loaded(struct amdgpu_device * adev)124b843c749SSergey Zigachev static int vce_v4_0_firmware_loaded(struct amdgpu_device *adev)
125b843c749SSergey Zigachev {
126b843c749SSergey Zigachev int i, j;
127b843c749SSergey Zigachev
128b843c749SSergey Zigachev for (i = 0; i < 10; ++i) {
129b843c749SSergey Zigachev for (j = 0; j < 100; ++j) {
130b843c749SSergey Zigachev uint32_t status =
131b843c749SSergey Zigachev RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS));
132b843c749SSergey Zigachev
133b843c749SSergey Zigachev if (status & VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK)
134b843c749SSergey Zigachev return 0;
135b843c749SSergey Zigachev mdelay(10);
136b843c749SSergey Zigachev }
137b843c749SSergey Zigachev
138b843c749SSergey Zigachev DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
139b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),
140b843c749SSergey Zigachev VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
141b843c749SSergey Zigachev ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
142b843c749SSergey Zigachev mdelay(10);
143b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET), 0,
144b843c749SSergey Zigachev ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
145b843c749SSergey Zigachev mdelay(10);
146b843c749SSergey Zigachev
147b843c749SSergey Zigachev }
148b843c749SSergey Zigachev
149b843c749SSergey Zigachev return -ETIMEDOUT;
150b843c749SSergey Zigachev }
151b843c749SSergey Zigachev
vce_v4_0_mmsch_start(struct amdgpu_device * adev,struct amdgpu_mm_table * table)152b843c749SSergey Zigachev static int vce_v4_0_mmsch_start(struct amdgpu_device *adev,
153b843c749SSergey Zigachev struct amdgpu_mm_table *table)
154b843c749SSergey Zigachev {
155b843c749SSergey Zigachev uint32_t data = 0, loop;
156b843c749SSergey Zigachev uint64_t addr = table->gpu_addr;
157b843c749SSergey Zigachev struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)table->cpu_addr;
158b843c749SSergey Zigachev uint32_t size;
159b843c749SSergey Zigachev
160b843c749SSergey Zigachev size = header->header_size + header->vce_table_size + header->uvd_table_size;
161b843c749SSergey Zigachev
162b843c749SSergey Zigachev /* 1, write to vce_mmsch_vf_ctx_addr_lo/hi register with GPU mc addr of memory descriptor location */
163b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_LO), lower_32_bits(addr));
164b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_ADDR_HI), upper_32_bits(addr));
165b843c749SSergey Zigachev
166b843c749SSergey Zigachev /* 2, update vmid of descriptor */
167b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID));
168b843c749SSergey Zigachev data &= ~VCE_MMSCH_VF_VMID__VF_CTX_VMID_MASK;
169b843c749SSergey Zigachev data |= (0 << VCE_MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); /* use domain0 for MM scheduler */
170b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_VMID), data);
171b843c749SSergey Zigachev
172b843c749SSergey Zigachev /* 3, notify mmsch about the size of this descriptor */
173b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_CTX_SIZE), size);
174b843c749SSergey Zigachev
175b843c749SSergey Zigachev /* 4, set resp to zero */
176b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP), 0);
177b843c749SSergey Zigachev
178b843c749SSergey Zigachev WDOORBELL32(adev->vce.ring[0].doorbell_index, 0);
179b843c749SSergey Zigachev adev->wb.wb[adev->vce.ring[0].wptr_offs] = 0;
180b843c749SSergey Zigachev adev->vce.ring[0].wptr = 0;
181b843c749SSergey Zigachev adev->vce.ring[0].wptr_old = 0;
182b843c749SSergey Zigachev
183b843c749SSergey Zigachev /* 5, kick off the initialization and wait until VCE_MMSCH_VF_MAILBOX_RESP becomes non-zero */
184b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_HOST), 0x10000001);
185b843c749SSergey Zigachev
186b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));
187b843c749SSergey Zigachev loop = 1000;
188b843c749SSergey Zigachev while ((data & 0x10000002) != 0x10000002) {
189b843c749SSergey Zigachev udelay(10);
190b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_MMSCH_VF_MAILBOX_RESP));
191b843c749SSergey Zigachev loop--;
192b843c749SSergey Zigachev if (!loop)
193b843c749SSergey Zigachev break;
194b843c749SSergey Zigachev }
195b843c749SSergey Zigachev
196b843c749SSergey Zigachev if (!loop) {
197b843c749SSergey Zigachev dev_err(adev->dev, "failed to init MMSCH, mmVCE_MMSCH_VF_MAILBOX_RESP = %x\n", data);
198b843c749SSergey Zigachev return -EBUSY;
199b843c749SSergey Zigachev }
200b843c749SSergey Zigachev
201b843c749SSergey Zigachev return 0;
202b843c749SSergey Zigachev }
203b843c749SSergey Zigachev
vce_v4_0_sriov_start(struct amdgpu_device * adev)204b843c749SSergey Zigachev static int vce_v4_0_sriov_start(struct amdgpu_device *adev)
205b843c749SSergey Zigachev {
206b843c749SSergey Zigachev struct amdgpu_ring *ring;
207b843c749SSergey Zigachev uint32_t offset, size;
208b843c749SSergey Zigachev uint32_t table_size = 0;
209b843c749SSergey Zigachev struct mmsch_v1_0_cmd_direct_write direct_wt = { { 0 } };
210b843c749SSergey Zigachev struct mmsch_v1_0_cmd_direct_read_modify_write direct_rd_mod_wt = { { 0 } };
211b843c749SSergey Zigachev struct mmsch_v1_0_cmd_direct_polling direct_poll = { { 0 } };
212b843c749SSergey Zigachev struct mmsch_v1_0_cmd_end end = { { 0 } };
213b843c749SSergey Zigachev uint32_t *init_table = adev->virt.mm_table.cpu_addr;
214b843c749SSergey Zigachev struct mmsch_v1_0_init_header *header = (struct mmsch_v1_0_init_header *)init_table;
215b843c749SSergey Zigachev
216b843c749SSergey Zigachev direct_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_WRITE;
217b843c749SSergey Zigachev direct_rd_mod_wt.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
218b843c749SSergey Zigachev direct_poll.cmd_header.command_type = MMSCH_COMMAND__DIRECT_REG_POLLING;
219b843c749SSergey Zigachev end.cmd_header.command_type = MMSCH_COMMAND__END;
220b843c749SSergey Zigachev
221b843c749SSergey Zigachev if (header->vce_table_offset == 0 && header->vce_table_size == 0) {
222b843c749SSergey Zigachev header->version = MMSCH_VERSION;
223b843c749SSergey Zigachev header->header_size = sizeof(struct mmsch_v1_0_init_header) >> 2;
224b843c749SSergey Zigachev
225b843c749SSergey Zigachev if (header->uvd_table_offset == 0 && header->uvd_table_size == 0)
226b843c749SSergey Zigachev header->vce_table_offset = header->header_size;
227b843c749SSergey Zigachev else
228b843c749SSergey Zigachev header->vce_table_offset = header->uvd_table_size + header->uvd_table_offset;
229b843c749SSergey Zigachev
230b843c749SSergey Zigachev init_table += header->vce_table_offset;
231b843c749SSergey Zigachev
232b843c749SSergey Zigachev ring = &adev->vce.ring[0];
233b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO),
234b843c749SSergey Zigachev lower_32_bits(ring->gpu_addr));
235b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI),
236b843c749SSergey Zigachev upper_32_bits(ring->gpu_addr));
237b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE),
238b843c749SSergey Zigachev ring->ring_size / 4);
239b843c749SSergey Zigachev
240b843c749SSergey Zigachev /* BEGING OF MC_RESUME */
241b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x398000);
242b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), ~0x1, 0);
243b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0);
244b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0);
245b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);
246b843c749SSergey Zigachev
247b843c749SSergey Zigachev if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
248b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
249b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
250b843c749SSergey Zigachev adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8);
251b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
252b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
253b843c749SSergey Zigachev (adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 40) & 0xff);
254b843c749SSergey Zigachev } else {
255b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
256b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
257b843c749SSergey Zigachev adev->vce.gpu_addr >> 8);
258b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
259b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
260b843c749SSergey Zigachev (adev->vce.gpu_addr >> 40) & 0xff);
261b843c749SSergey Zigachev }
262b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
263b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_40BIT_BAR1),
264b843c749SSergey Zigachev adev->vce.gpu_addr >> 8);
265b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
266b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_64BIT_BAR1),
267b843c749SSergey Zigachev (adev->vce.gpu_addr >> 40) & 0xff);
268b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
269b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_40BIT_BAR2),
270b843c749SSergey Zigachev adev->vce.gpu_addr >> 8);
271b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0,
272b843c749SSergey Zigachev mmVCE_LMI_VCPU_CACHE_64BIT_BAR2),
273b843c749SSergey Zigachev (adev->vce.gpu_addr >> 40) & 0xff);
274b843c749SSergey Zigachev
275b843c749SSergey Zigachev offset = AMDGPU_VCE_FIRMWARE_OFFSET;
276b843c749SSergey Zigachev size = VCE_V4_0_FW_SIZE;
277b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0),
278b843c749SSergey Zigachev offset & ~0x0f000000);
279b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);
280b843c749SSergey Zigachev
281b843c749SSergey Zigachev offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;
282b843c749SSergey Zigachev size = VCE_V4_0_STACK_SIZE;
283b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1),
284b843c749SSergey Zigachev (offset & ~0x0f000000) | (1 << 24));
285b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);
286b843c749SSergey Zigachev
287b843c749SSergey Zigachev offset += size;
288b843c749SSergey Zigachev size = VCE_V4_0_DATA_SIZE;
289b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2),
290b843c749SSergey Zigachev (offset & ~0x0f000000) | (2 << 24));
291b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);
292b843c749SSergey Zigachev
293b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), ~0x100, 0);
294b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),
295b843c749SSergey Zigachev VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
296b843c749SSergey Zigachev VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
297b843c749SSergey Zigachev
298b843c749SSergey Zigachev /* end of MC_RESUME */
299b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
300b843c749SSergey Zigachev VCE_STATUS__JOB_BUSY_MASK, ~VCE_STATUS__JOB_BUSY_MASK);
301b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL),
302b843c749SSergey Zigachev ~0x200001, VCE_VCPU_CNTL__CLK_EN_MASK);
303b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),
304b843c749SSergey Zigachev ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK, 0);
305b843c749SSergey Zigachev
306b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_POLL(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
307b843c749SSergey Zigachev VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK,
308b843c749SSergey Zigachev VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK);
309b843c749SSergey Zigachev
310b843c749SSergey Zigachev /* clear BUSY flag */
311b843c749SSergey Zigachev MMSCH_V1_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS),
312b843c749SSergey Zigachev ~VCE_STATUS__JOB_BUSY_MASK, 0);
313b843c749SSergey Zigachev
314b843c749SSergey Zigachev /* add end packet */
315b843c749SSergey Zigachev memcpy((void *)init_table, &end, sizeof(struct mmsch_v1_0_cmd_end));
316b843c749SSergey Zigachev table_size += sizeof(struct mmsch_v1_0_cmd_end) / 4;
317b843c749SSergey Zigachev header->vce_table_size = table_size;
318b843c749SSergey Zigachev }
319b843c749SSergey Zigachev
320b843c749SSergey Zigachev return vce_v4_0_mmsch_start(adev, &adev->virt.mm_table);
321b843c749SSergey Zigachev }
322b843c749SSergey Zigachev
323b843c749SSergey Zigachev /**
324b843c749SSergey Zigachev * vce_v4_0_start - start VCE block
325b843c749SSergey Zigachev *
326b843c749SSergey Zigachev * @adev: amdgpu_device pointer
327b843c749SSergey Zigachev *
328b843c749SSergey Zigachev * Setup and start the VCE block
329b843c749SSergey Zigachev */
vce_v4_0_start(struct amdgpu_device * adev)330b843c749SSergey Zigachev static int vce_v4_0_start(struct amdgpu_device *adev)
331b843c749SSergey Zigachev {
332b843c749SSergey Zigachev struct amdgpu_ring *ring;
333b843c749SSergey Zigachev int r;
334b843c749SSergey Zigachev
335b843c749SSergey Zigachev ring = &adev->vce.ring[0];
336b843c749SSergey Zigachev
337b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR), lower_32_bits(ring->wptr));
338b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR), lower_32_bits(ring->wptr));
339b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO), ring->gpu_addr);
340b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI), upper_32_bits(ring->gpu_addr));
341b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE), ring->ring_size / 4);
342b843c749SSergey Zigachev
343b843c749SSergey Zigachev ring = &adev->vce.ring[1];
344b843c749SSergey Zigachev
345b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR2), lower_32_bits(ring->wptr));
346b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR2), lower_32_bits(ring->wptr));
347b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO2), ring->gpu_addr);
348b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI2), upper_32_bits(ring->gpu_addr));
349b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE2), ring->ring_size / 4);
350b843c749SSergey Zigachev
351b843c749SSergey Zigachev ring = &adev->vce.ring[2];
352b843c749SSergey Zigachev
353b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_RPTR3), lower_32_bits(ring->wptr));
354b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_WPTR3), lower_32_bits(ring->wptr));
355b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_LO3), ring->gpu_addr);
356b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_BASE_HI3), upper_32_bits(ring->gpu_addr));
357b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_SIZE3), ring->ring_size / 4);
358b843c749SSergey Zigachev
359b843c749SSergey Zigachev vce_v4_0_mc_resume(adev);
360b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), VCE_STATUS__JOB_BUSY_MASK,
361b843c749SSergey Zigachev ~VCE_STATUS__JOB_BUSY_MASK);
362b843c749SSergey Zigachev
363b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 1, ~0x200001);
364b843c749SSergey Zigachev
365b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET), 0,
366b843c749SSergey Zigachev ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
367b843c749SSergey Zigachev mdelay(100);
368b843c749SSergey Zigachev
369b843c749SSergey Zigachev r = vce_v4_0_firmware_loaded(adev);
370b843c749SSergey Zigachev
371b843c749SSergey Zigachev /* clear BUSY flag */
372b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0, ~VCE_STATUS__JOB_BUSY_MASK);
373b843c749SSergey Zigachev
374b843c749SSergey Zigachev if (r) {
375b843c749SSergey Zigachev DRM_ERROR("VCE not responding, giving up!!!\n");
376b843c749SSergey Zigachev return r;
377b843c749SSergey Zigachev }
378b843c749SSergey Zigachev
379b843c749SSergey Zigachev return 0;
380b843c749SSergey Zigachev }
381b843c749SSergey Zigachev
vce_v4_0_stop(struct amdgpu_device * adev)382b843c749SSergey Zigachev static int vce_v4_0_stop(struct amdgpu_device *adev)
383b843c749SSergey Zigachev {
384b843c749SSergey Zigachev
385b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CNTL), 0, ~0x200001);
386b843c749SSergey Zigachev
387b843c749SSergey Zigachev /* hold on ECPU */
388b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SOFT_RESET),
389b843c749SSergey Zigachev VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
390b843c749SSergey Zigachev ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
391b843c749SSergey Zigachev
392b843c749SSergey Zigachev /* clear BUSY flag */
393b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS), 0, ~VCE_STATUS__JOB_BUSY_MASK);
394b843c749SSergey Zigachev
395b843c749SSergey Zigachev /* Set Clock-Gating off */
396b843c749SSergey Zigachev /* if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)
397b843c749SSergey Zigachev vce_v4_0_set_vce_sw_clock_gating(adev, false);
398b843c749SSergey Zigachev */
399b843c749SSergey Zigachev
400b843c749SSergey Zigachev return 0;
401b843c749SSergey Zigachev }
402b843c749SSergey Zigachev
vce_v4_0_early_init(void * handle)403b843c749SSergey Zigachev static int vce_v4_0_early_init(void *handle)
404b843c749SSergey Zigachev {
405b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
406b843c749SSergey Zigachev
407b843c749SSergey Zigachev if (amdgpu_sriov_vf(adev)) /* currently only VCN0 support SRIOV */
408b843c749SSergey Zigachev adev->vce.num_rings = 1;
409b843c749SSergey Zigachev else
410b843c749SSergey Zigachev adev->vce.num_rings = 3;
411b843c749SSergey Zigachev
412b843c749SSergey Zigachev vce_v4_0_set_ring_funcs(adev);
413b843c749SSergey Zigachev vce_v4_0_set_irq_funcs(adev);
414b843c749SSergey Zigachev
415b843c749SSergey Zigachev return 0;
416b843c749SSergey Zigachev }
417b843c749SSergey Zigachev
vce_v4_0_sw_init(void * handle)418b843c749SSergey Zigachev static int vce_v4_0_sw_init(void *handle)
419b843c749SSergey Zigachev {
420b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
421b843c749SSergey Zigachev struct amdgpu_ring *ring;
422b843c749SSergey Zigachev
423b843c749SSergey Zigachev unsigned size;
424b843c749SSergey Zigachev int r, i;
425b843c749SSergey Zigachev
426b843c749SSergey Zigachev r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCE0, 167, &adev->vce.irq);
427b843c749SSergey Zigachev if (r)
428b843c749SSergey Zigachev return r;
429b843c749SSergey Zigachev
430b843c749SSergey Zigachev size = VCE_V4_0_STACK_SIZE + VCE_V4_0_DATA_SIZE;
431b843c749SSergey Zigachev if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
432b843c749SSergey Zigachev size += VCE_V4_0_FW_SIZE;
433b843c749SSergey Zigachev
434b843c749SSergey Zigachev r = amdgpu_vce_sw_init(adev, size);
435b843c749SSergey Zigachev if (r)
436b843c749SSergey Zigachev return r;
437b843c749SSergey Zigachev
438b843c749SSergey Zigachev if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
439b843c749SSergey Zigachev const struct common_firmware_header *hdr;
440b843c749SSergey Zigachev unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
441b843c749SSergey Zigachev
442*78973132SSergey Zigachev adev->vce.saved_bo = kmalloc(size, M_DRM, GFP_KERNEL);
443b843c749SSergey Zigachev if (!adev->vce.saved_bo)
444b843c749SSergey Zigachev return -ENOMEM;
445b843c749SSergey Zigachev
446b843c749SSergey Zigachev hdr = (const struct common_firmware_header *)adev->vce.fw->data;
447b843c749SSergey Zigachev adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].ucode_id = AMDGPU_UCODE_ID_VCE;
448b843c749SSergey Zigachev adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].fw = adev->vce.fw;
449b843c749SSergey Zigachev adev->firmware.fw_size +=
450b843c749SSergey Zigachev ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE);
451b843c749SSergey Zigachev DRM_INFO("PSP loading VCE firmware\n");
452b843c749SSergey Zigachev } else {
453b843c749SSergey Zigachev r = amdgpu_vce_resume(adev);
454b843c749SSergey Zigachev if (r)
455b843c749SSergey Zigachev return r;
456b843c749SSergey Zigachev }
457b843c749SSergey Zigachev
458b843c749SSergey Zigachev for (i = 0; i < adev->vce.num_rings; i++) {
459b843c749SSergey Zigachev ring = &adev->vce.ring[i];
460*78973132SSergey Zigachev ksprintf(ring->name, "vce%d", i);
461b843c749SSergey Zigachev if (amdgpu_sriov_vf(adev)) {
462b843c749SSergey Zigachev /* DOORBELL only works under SRIOV */
463b843c749SSergey Zigachev ring->use_doorbell = true;
464b843c749SSergey Zigachev
465b843c749SSergey Zigachev /* currently only use the first encoding ring for sriov,
466b843c749SSergey Zigachev * so set unused location for other unused rings.
467b843c749SSergey Zigachev */
468b843c749SSergey Zigachev if (i == 0)
469b843c749SSergey Zigachev ring->doorbell_index = AMDGPU_DOORBELL64_VCE_RING0_1 * 2;
470b843c749SSergey Zigachev else
471b843c749SSergey Zigachev ring->doorbell_index = AMDGPU_DOORBELL64_VCE_RING2_3 * 2 + 1;
472b843c749SSergey Zigachev }
473b843c749SSergey Zigachev r = amdgpu_ring_init(adev, ring, 512, &adev->vce.irq, 0);
474b843c749SSergey Zigachev if (r)
475b843c749SSergey Zigachev return r;
476b843c749SSergey Zigachev }
477b843c749SSergey Zigachev
478b843c749SSergey Zigachev
479b843c749SSergey Zigachev r = amdgpu_vce_entity_init(adev);
480b843c749SSergey Zigachev if (r)
481b843c749SSergey Zigachev return r;
482b843c749SSergey Zigachev
483b843c749SSergey Zigachev r = amdgpu_virt_alloc_mm_table(adev);
484b843c749SSergey Zigachev if (r)
485b843c749SSergey Zigachev return r;
486b843c749SSergey Zigachev
487b843c749SSergey Zigachev return r;
488b843c749SSergey Zigachev }
489b843c749SSergey Zigachev
vce_v4_0_sw_fini(void * handle)490b843c749SSergey Zigachev static int vce_v4_0_sw_fini(void *handle)
491b843c749SSergey Zigachev {
492b843c749SSergey Zigachev int r;
493b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
494b843c749SSergey Zigachev
495b843c749SSergey Zigachev /* free MM table */
496b843c749SSergey Zigachev amdgpu_virt_free_mm_table(adev);
497b843c749SSergey Zigachev
498b843c749SSergey Zigachev if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
499b843c749SSergey Zigachev kvfree(adev->vce.saved_bo);
500b843c749SSergey Zigachev adev->vce.saved_bo = NULL;
501b843c749SSergey Zigachev }
502b843c749SSergey Zigachev
503b843c749SSergey Zigachev r = amdgpu_vce_suspend(adev);
504b843c749SSergey Zigachev if (r)
505b843c749SSergey Zigachev return r;
506b843c749SSergey Zigachev
507b843c749SSergey Zigachev return amdgpu_vce_sw_fini(adev);
508b843c749SSergey Zigachev }
509b843c749SSergey Zigachev
vce_v4_0_hw_init(void * handle)510b843c749SSergey Zigachev static int vce_v4_0_hw_init(void *handle)
511b843c749SSergey Zigachev {
512b843c749SSergey Zigachev int r, i;
513b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
514b843c749SSergey Zigachev
515b843c749SSergey Zigachev if (amdgpu_sriov_vf(adev))
516b843c749SSergey Zigachev r = vce_v4_0_sriov_start(adev);
517b843c749SSergey Zigachev else
518b843c749SSergey Zigachev r = vce_v4_0_start(adev);
519b843c749SSergey Zigachev if (r)
520b843c749SSergey Zigachev return r;
521b843c749SSergey Zigachev
522b843c749SSergey Zigachev for (i = 0; i < adev->vce.num_rings; i++)
523b843c749SSergey Zigachev adev->vce.ring[i].ready = false;
524b843c749SSergey Zigachev
525b843c749SSergey Zigachev for (i = 0; i < adev->vce.num_rings; i++) {
526b843c749SSergey Zigachev r = amdgpu_ring_test_ring(&adev->vce.ring[i]);
527b843c749SSergey Zigachev if (r)
528b843c749SSergey Zigachev return r;
529b843c749SSergey Zigachev else
530b843c749SSergey Zigachev adev->vce.ring[i].ready = true;
531b843c749SSergey Zigachev }
532b843c749SSergey Zigachev
533b843c749SSergey Zigachev DRM_INFO("VCE initialized successfully.\n");
534b843c749SSergey Zigachev
535b843c749SSergey Zigachev return 0;
536b843c749SSergey Zigachev }
537b843c749SSergey Zigachev
vce_v4_0_hw_fini(void * handle)538b843c749SSergey Zigachev static int vce_v4_0_hw_fini(void *handle)
539b843c749SSergey Zigachev {
540b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
541b843c749SSergey Zigachev int i;
542b843c749SSergey Zigachev
543b843c749SSergey Zigachev if (!amdgpu_sriov_vf(adev)) {
544b843c749SSergey Zigachev /* vce_v4_0_wait_for_idle(handle); */
545b843c749SSergey Zigachev vce_v4_0_stop(adev);
546b843c749SSergey Zigachev } else {
547b843c749SSergey Zigachev /* full access mode, so don't touch any VCE register */
548b843c749SSergey Zigachev DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
549b843c749SSergey Zigachev }
550b843c749SSergey Zigachev
551b843c749SSergey Zigachev for (i = 0; i < adev->vce.num_rings; i++)
552b843c749SSergey Zigachev adev->vce.ring[i].ready = false;
553b843c749SSergey Zigachev
554b843c749SSergey Zigachev return 0;
555b843c749SSergey Zigachev }
556b843c749SSergey Zigachev
vce_v4_0_suspend(void * handle)557b843c749SSergey Zigachev static int vce_v4_0_suspend(void *handle)
558b843c749SSergey Zigachev {
559b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
560b843c749SSergey Zigachev int r;
561b843c749SSergey Zigachev
562b843c749SSergey Zigachev if (adev->vce.vcpu_bo == NULL)
563b843c749SSergey Zigachev return 0;
564b843c749SSergey Zigachev
565b843c749SSergey Zigachev if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
566b843c749SSergey Zigachev unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
567b843c749SSergey Zigachev void *ptr = adev->vce.cpu_addr;
568b843c749SSergey Zigachev
569b843c749SSergey Zigachev memcpy_fromio(adev->vce.saved_bo, ptr, size);
570b843c749SSergey Zigachev }
571b843c749SSergey Zigachev
572b843c749SSergey Zigachev r = vce_v4_0_hw_fini(adev);
573b843c749SSergey Zigachev if (r)
574b843c749SSergey Zigachev return r;
575b843c749SSergey Zigachev
576b843c749SSergey Zigachev return amdgpu_vce_suspend(adev);
577b843c749SSergey Zigachev }
578b843c749SSergey Zigachev
vce_v4_0_resume(void * handle)579b843c749SSergey Zigachev static int vce_v4_0_resume(void *handle)
580b843c749SSergey Zigachev {
581b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
582b843c749SSergey Zigachev int r;
583b843c749SSergey Zigachev
584b843c749SSergey Zigachev if (adev->vce.vcpu_bo == NULL)
585b843c749SSergey Zigachev return -EINVAL;
586b843c749SSergey Zigachev
587b843c749SSergey Zigachev if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
588b843c749SSergey Zigachev unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
589b843c749SSergey Zigachev void *ptr = adev->vce.cpu_addr;
590b843c749SSergey Zigachev
591b843c749SSergey Zigachev memcpy_toio(ptr, adev->vce.saved_bo, size);
592b843c749SSergey Zigachev } else {
593b843c749SSergey Zigachev r = amdgpu_vce_resume(adev);
594b843c749SSergey Zigachev if (r)
595b843c749SSergey Zigachev return r;
596b843c749SSergey Zigachev }
597b843c749SSergey Zigachev
598b843c749SSergey Zigachev return vce_v4_0_hw_init(adev);
599b843c749SSergey Zigachev }
600b843c749SSergey Zigachev
vce_v4_0_mc_resume(struct amdgpu_device * adev)601b843c749SSergey Zigachev static void vce_v4_0_mc_resume(struct amdgpu_device *adev)
602b843c749SSergey Zigachev {
603b843c749SSergey Zigachev uint32_t offset, size;
604b843c749SSergey Zigachev
605b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A), 0, ~(1 << 16));
606b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), 0x1FF000, ~0xFF9FF000);
607b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), 0x3F, ~0x3F);
608b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), 0x1FF);
609b843c749SSergey Zigachev
610b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL), 0x00398000);
611b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CACHE_CTRL), 0x0, ~0x1);
612b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL), 0);
613b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_SWAP_CNTL1), 0);
614b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VM_CTRL), 0);
615b843c749SSergey Zigachev
616b843c749SSergey Zigachev if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
617b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
618b843c749SSergey Zigachev (adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 8));
619b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
620b843c749SSergey Zigachev (adev->firmware.ucode[AMDGPU_UCODE_ID_VCE].mc_addr >> 40) & 0xff);
621b843c749SSergey Zigachev } else {
622b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR0),
623b843c749SSergey Zigachev (adev->vce.gpu_addr >> 8));
624b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR0),
625b843c749SSergey Zigachev (adev->vce.gpu_addr >> 40) & 0xff);
626b843c749SSergey Zigachev }
627b843c749SSergey Zigachev
628b843c749SSergey Zigachev offset = AMDGPU_VCE_FIRMWARE_OFFSET;
629b843c749SSergey Zigachev size = VCE_V4_0_FW_SIZE;
630b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET0), offset & ~0x0f000000);
631b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE0), size);
632b843c749SSergey Zigachev
633b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR1), (adev->vce.gpu_addr >> 8));
634b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR1), (adev->vce.gpu_addr >> 40) & 0xff);
635b843c749SSergey Zigachev offset = (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) ? offset + size : 0;
636b843c749SSergey Zigachev size = VCE_V4_0_STACK_SIZE;
637b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET1), (offset & ~0x0f000000) | (1 << 24));
638b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE1), size);
639b843c749SSergey Zigachev
640b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_40BIT_BAR2), (adev->vce.gpu_addr >> 8));
641b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_VCPU_CACHE_64BIT_BAR2), (adev->vce.gpu_addr >> 40) & 0xff);
642b843c749SSergey Zigachev offset += size;
643b843c749SSergey Zigachev size = VCE_V4_0_DATA_SIZE;
644b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_OFFSET2), (offset & ~0x0f000000) | (2 << 24));
645b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_VCPU_CACHE_SIZE2), size);
646b843c749SSergey Zigachev
647b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_LMI_CTRL2), 0x0, ~0x100);
648b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN),
649b843c749SSergey Zigachev VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK,
650b843c749SSergey Zigachev ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
651b843c749SSergey Zigachev }
652b843c749SSergey Zigachev
vce_v4_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)653b843c749SSergey Zigachev static int vce_v4_0_set_clockgating_state(void *handle,
654b843c749SSergey Zigachev enum amd_clockgating_state state)
655b843c749SSergey Zigachev {
656b843c749SSergey Zigachev /* needed for driver unload*/
657b843c749SSergey Zigachev return 0;
658b843c749SSergey Zigachev }
659b843c749SSergey Zigachev
660b843c749SSergey Zigachev #if 0
661b843c749SSergey Zigachev static bool vce_v4_0_is_idle(void *handle)
662b843c749SSergey Zigachev {
663b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
664b843c749SSergey Zigachev u32 mask = 0;
665b843c749SSergey Zigachev
666b843c749SSergey Zigachev mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE0) ? 0 : SRBM_STATUS2__VCE0_BUSY_MASK;
667b843c749SSergey Zigachev mask |= (adev->vce.harvest_config & AMDGPU_VCE_HARVEST_VCE1) ? 0 : SRBM_STATUS2__VCE1_BUSY_MASK;
668b843c749SSergey Zigachev
669b843c749SSergey Zigachev return !(RREG32(mmSRBM_STATUS2) & mask);
670b843c749SSergey Zigachev }
671b843c749SSergey Zigachev
672b843c749SSergey Zigachev static int vce_v4_0_wait_for_idle(void *handle)
673b843c749SSergey Zigachev {
674b843c749SSergey Zigachev unsigned i;
675b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
676b843c749SSergey Zigachev
677b843c749SSergey Zigachev for (i = 0; i < adev->usec_timeout; i++)
678b843c749SSergey Zigachev if (vce_v4_0_is_idle(handle))
679b843c749SSergey Zigachev return 0;
680b843c749SSergey Zigachev
681b843c749SSergey Zigachev return -ETIMEDOUT;
682b843c749SSergey Zigachev }
683b843c749SSergey Zigachev
684b843c749SSergey Zigachev #define VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK 0x00000008L /* AUTO_BUSY */
685b843c749SSergey Zigachev #define VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK 0x00000010L /* RB0_BUSY */
686b843c749SSergey Zigachev #define VCE_STATUS_VCPU_REPORT_RB1_BUSY_MASK 0x00000020L /* RB1_BUSY */
687b843c749SSergey Zigachev #define AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \
688b843c749SSergey Zigachev VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK)
689b843c749SSergey Zigachev
690b843c749SSergey Zigachev static bool vce_v4_0_check_soft_reset(void *handle)
691b843c749SSergey Zigachev {
692b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
693b843c749SSergey Zigachev u32 srbm_soft_reset = 0;
694b843c749SSergey Zigachev
695b843c749SSergey Zigachev /* According to VCE team , we should use VCE_STATUS instead
696b843c749SSergey Zigachev * SRBM_STATUS.VCE_BUSY bit for busy status checking.
697b843c749SSergey Zigachev * GRBM_GFX_INDEX.INSTANCE_INDEX is used to specify which VCE
698b843c749SSergey Zigachev * instance's registers are accessed
699b843c749SSergey Zigachev * (0 for 1st instance, 10 for 2nd instance).
700b843c749SSergey Zigachev *
701b843c749SSergey Zigachev *VCE_STATUS
702b843c749SSergey Zigachev *|UENC|ACPI|AUTO ACTIVE|RB1 |RB0 |RB2 | |FW_LOADED|JOB |
703b843c749SSergey Zigachev *|----+----+-----------+----+----+----+----------+---------+----|
704b843c749SSergey Zigachev *|bit8|bit7| bit6 |bit5|bit4|bit3| bit2 | bit1 |bit0|
705b843c749SSergey Zigachev *
706b843c749SSergey Zigachev * VCE team suggest use bit 3--bit 6 for busy status check
707b843c749SSergey Zigachev */
708b843c749SSergey Zigachev mutex_lock(&adev->grbm_idx_mutex);
709b843c749SSergey Zigachev WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);
710b843c749SSergey Zigachev if (RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
711b843c749SSergey Zigachev srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
712b843c749SSergey Zigachev srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
713b843c749SSergey Zigachev }
714b843c749SSergey Zigachev WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0x10);
715b843c749SSergey Zigachev if (RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) {
716b843c749SSergey Zigachev srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1);
717b843c749SSergey Zigachev srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1);
718b843c749SSergey Zigachev }
719b843c749SSergey Zigachev WREG32_FIELD(GRBM_GFX_INDEX, INSTANCE_INDEX, 0);
720b843c749SSergey Zigachev mutex_unlock(&adev->grbm_idx_mutex);
721b843c749SSergey Zigachev
722b843c749SSergey Zigachev if (srbm_soft_reset) {
723b843c749SSergey Zigachev adev->vce.srbm_soft_reset = srbm_soft_reset;
724b843c749SSergey Zigachev return true;
725b843c749SSergey Zigachev } else {
726b843c749SSergey Zigachev adev->vce.srbm_soft_reset = 0;
727b843c749SSergey Zigachev return false;
728b843c749SSergey Zigachev }
729b843c749SSergey Zigachev }
730b843c749SSergey Zigachev
731b843c749SSergey Zigachev static int vce_v4_0_soft_reset(void *handle)
732b843c749SSergey Zigachev {
733b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
734b843c749SSergey Zigachev u32 srbm_soft_reset;
735b843c749SSergey Zigachev
736b843c749SSergey Zigachev if (!adev->vce.srbm_soft_reset)
737b843c749SSergey Zigachev return 0;
738b843c749SSergey Zigachev srbm_soft_reset = adev->vce.srbm_soft_reset;
739b843c749SSergey Zigachev
740b843c749SSergey Zigachev if (srbm_soft_reset) {
741b843c749SSergey Zigachev u32 tmp;
742b843c749SSergey Zigachev
743b843c749SSergey Zigachev tmp = RREG32(mmSRBM_SOFT_RESET);
744b843c749SSergey Zigachev tmp |= srbm_soft_reset;
745b843c749SSergey Zigachev dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
746b843c749SSergey Zigachev WREG32(mmSRBM_SOFT_RESET, tmp);
747b843c749SSergey Zigachev tmp = RREG32(mmSRBM_SOFT_RESET);
748b843c749SSergey Zigachev
749b843c749SSergey Zigachev udelay(50);
750b843c749SSergey Zigachev
751b843c749SSergey Zigachev tmp &= ~srbm_soft_reset;
752b843c749SSergey Zigachev WREG32(mmSRBM_SOFT_RESET, tmp);
753b843c749SSergey Zigachev tmp = RREG32(mmSRBM_SOFT_RESET);
754b843c749SSergey Zigachev
755b843c749SSergey Zigachev /* Wait a little for things to settle down */
756b843c749SSergey Zigachev udelay(50);
757b843c749SSergey Zigachev }
758b843c749SSergey Zigachev
759b843c749SSergey Zigachev return 0;
760b843c749SSergey Zigachev }
761b843c749SSergey Zigachev
762b843c749SSergey Zigachev static int vce_v4_0_pre_soft_reset(void *handle)
763b843c749SSergey Zigachev {
764b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
765b843c749SSergey Zigachev
766b843c749SSergey Zigachev if (!adev->vce.srbm_soft_reset)
767b843c749SSergey Zigachev return 0;
768b843c749SSergey Zigachev
769b843c749SSergey Zigachev mdelay(5);
770b843c749SSergey Zigachev
771b843c749SSergey Zigachev return vce_v4_0_suspend(adev);
772b843c749SSergey Zigachev }
773b843c749SSergey Zigachev
774b843c749SSergey Zigachev
775b843c749SSergey Zigachev static int vce_v4_0_post_soft_reset(void *handle)
776b843c749SSergey Zigachev {
777b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
778b843c749SSergey Zigachev
779b843c749SSergey Zigachev if (!adev->vce.srbm_soft_reset)
780b843c749SSergey Zigachev return 0;
781b843c749SSergey Zigachev
782b843c749SSergey Zigachev mdelay(5);
783b843c749SSergey Zigachev
784b843c749SSergey Zigachev return vce_v4_0_resume(adev);
785b843c749SSergey Zigachev }
786b843c749SSergey Zigachev
787b843c749SSergey Zigachev static void vce_v4_0_override_vce_clock_gating(struct amdgpu_device *adev, bool override)
788b843c749SSergey Zigachev {
789b843c749SSergey Zigachev u32 tmp, data;
790b843c749SSergey Zigachev
791b843c749SSergey Zigachev tmp = data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_ARB_CTRL));
792b843c749SSergey Zigachev if (override)
793b843c749SSergey Zigachev data |= VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
794b843c749SSergey Zigachev else
795b843c749SSergey Zigachev data &= ~VCE_RB_ARB_CTRL__VCE_CGTT_OVERRIDE_MASK;
796b843c749SSergey Zigachev
797b843c749SSergey Zigachev if (tmp != data)
798b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_RB_ARB_CTRL), data);
799b843c749SSergey Zigachev }
800b843c749SSergey Zigachev
801b843c749SSergey Zigachev static void vce_v4_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
802b843c749SSergey Zigachev bool gated)
803b843c749SSergey Zigachev {
804b843c749SSergey Zigachev u32 data;
805b843c749SSergey Zigachev
806b843c749SSergey Zigachev /* Set Override to disable Clock Gating */
807b843c749SSergey Zigachev vce_v4_0_override_vce_clock_gating(adev, true);
808b843c749SSergey Zigachev
809b843c749SSergey Zigachev /* This function enables MGCG which is controlled by firmware.
810b843c749SSergey Zigachev With the clocks in the gated state the core is still
811b843c749SSergey Zigachev accessible but the firmware will throttle the clocks on the
812b843c749SSergey Zigachev fly as necessary.
813b843c749SSergey Zigachev */
814b843c749SSergey Zigachev if (gated) {
815b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B));
816b843c749SSergey Zigachev data |= 0x1ff;
817b843c749SSergey Zigachev data &= ~0xef0000;
818b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), data);
819b843c749SSergey Zigachev
820b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING));
821b843c749SSergey Zigachev data |= 0x3ff000;
822b843c749SSergey Zigachev data &= ~0xffc00000;
823b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), data);
824b843c749SSergey Zigachev
825b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2));
826b843c749SSergey Zigachev data |= 0x2;
827b843c749SSergey Zigachev data &= ~0x00010000;
828b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2), data);
829b843c749SSergey Zigachev
830b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING));
831b843c749SSergey Zigachev data |= 0x37f;
832b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), data);
833b843c749SSergey Zigachev
834b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL));
835b843c749SSergey Zigachev data |= VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
836b843c749SSergey Zigachev VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
837b843c749SSergey Zigachev VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK |
838b843c749SSergey Zigachev 0x8;
839b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL), data);
840b843c749SSergey Zigachev } else {
841b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B));
842b843c749SSergey Zigachev data &= ~0x80010;
843b843c749SSergey Zigachev data |= 0xe70008;
844b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_B), data);
845b843c749SSergey Zigachev
846b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING));
847b843c749SSergey Zigachev data |= 0xffc00000;
848b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING), data);
849b843c749SSergey Zigachev
850b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2));
851b843c749SSergey Zigachev data |= 0x10000;
852b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING_2), data);
853b843c749SSergey Zigachev
854b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING));
855b843c749SSergey Zigachev data &= ~0xffc00000;
856b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_REG_CLOCK_GATING), data);
857b843c749SSergey Zigachev
858b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL));
859b843c749SSergey Zigachev data &= ~(VCE_UENC_DMA_DCLK_CTRL__WRDMCLK_FORCEON_MASK |
860b843c749SSergey Zigachev VCE_UENC_DMA_DCLK_CTRL__RDDMCLK_FORCEON_MASK |
861b843c749SSergey Zigachev VCE_UENC_DMA_DCLK_CTRL__REGCLK_FORCEON_MASK |
862b843c749SSergey Zigachev 0x8);
863b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_DMA_DCLK_CTRL), data);
864b843c749SSergey Zigachev }
865b843c749SSergey Zigachev vce_v4_0_override_vce_clock_gating(adev, false);
866b843c749SSergey Zigachev }
867b843c749SSergey Zigachev
868b843c749SSergey Zigachev static void vce_v4_0_set_bypass_mode(struct amdgpu_device *adev, bool enable)
869b843c749SSergey Zigachev {
870b843c749SSergey Zigachev u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
871b843c749SSergey Zigachev
872b843c749SSergey Zigachev if (enable)
873b843c749SSergey Zigachev tmp |= GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
874b843c749SSergey Zigachev else
875b843c749SSergey Zigachev tmp &= ~GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
876b843c749SSergey Zigachev
877b843c749SSergey Zigachev WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);
878b843c749SSergey Zigachev }
879b843c749SSergey Zigachev
880b843c749SSergey Zigachev static int vce_v4_0_set_clockgating_state(void *handle,
881b843c749SSergey Zigachev enum amd_clockgating_state state)
882b843c749SSergey Zigachev {
883b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
884b843c749SSergey Zigachev bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
885b843c749SSergey Zigachev int i;
886b843c749SSergey Zigachev
887b843c749SSergey Zigachev if ((adev->asic_type == CHIP_POLARIS10) ||
888b843c749SSergey Zigachev (adev->asic_type == CHIP_TONGA) ||
889b843c749SSergey Zigachev (adev->asic_type == CHIP_FIJI))
890b843c749SSergey Zigachev vce_v4_0_set_bypass_mode(adev, enable);
891b843c749SSergey Zigachev
892b843c749SSergey Zigachev if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG))
893b843c749SSergey Zigachev return 0;
894b843c749SSergey Zigachev
895b843c749SSergey Zigachev mutex_lock(&adev->grbm_idx_mutex);
896b843c749SSergey Zigachev for (i = 0; i < 2; i++) {
897b843c749SSergey Zigachev /* Program VCE Instance 0 or 1 if not harvested */
898b843c749SSergey Zigachev if (adev->vce.harvest_config & (1 << i))
899b843c749SSergey Zigachev continue;
900b843c749SSergey Zigachev
901b843c749SSergey Zigachev WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, i);
902b843c749SSergey Zigachev
903b843c749SSergey Zigachev if (enable) {
904b843c749SSergey Zigachev /* initialize VCE_CLOCK_GATING_A: Clock ON/OFF delay */
905b843c749SSergey Zigachev uint32_t data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A);
906b843c749SSergey Zigachev data &= ~(0xf | 0xff0);
907b843c749SSergey Zigachev data |= ((0x0 << 0) | (0x04 << 4));
908b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_CLOCK_GATING_A, data);
909b843c749SSergey Zigachev
910b843c749SSergey Zigachev /* initialize VCE_UENC_CLOCK_GATING: Clock ON/OFF delay */
911b843c749SSergey Zigachev data = RREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING);
912b843c749SSergey Zigachev data &= ~(0xf | 0xff0);
913b843c749SSergey Zigachev data |= ((0x0 << 0) | (0x04 << 4));
914b843c749SSergey Zigachev WREG32(SOC15_REG_OFFSET(VCE, 0, mmVCE_UENC_CLOCK_GATING, data);
915b843c749SSergey Zigachev }
916b843c749SSergey Zigachev
917b843c749SSergey Zigachev vce_v4_0_set_vce_sw_clock_gating(adev, enable);
918b843c749SSergey Zigachev }
919b843c749SSergey Zigachev
920b843c749SSergey Zigachev WREG32_FIELD(GRBM_GFX_INDEX, VCE_INSTANCE, 0);
921b843c749SSergey Zigachev mutex_unlock(&adev->grbm_idx_mutex);
922b843c749SSergey Zigachev
923b843c749SSergey Zigachev return 0;
924b843c749SSergey Zigachev }
925b843c749SSergey Zigachev
926b843c749SSergey Zigachev static int vce_v4_0_set_powergating_state(void *handle,
927b843c749SSergey Zigachev enum amd_powergating_state state)
928b843c749SSergey Zigachev {
929b843c749SSergey Zigachev /* This doesn't actually powergate the VCE block.
930b843c749SSergey Zigachev * That's done in the dpm code via the SMC. This
931b843c749SSergey Zigachev * just re-inits the block as necessary. The actual
932b843c749SSergey Zigachev * gating still happens in the dpm code. We should
933b843c749SSergey Zigachev * revisit this when there is a cleaner line between
934b843c749SSergey Zigachev * the smc and the hw blocks
935b843c749SSergey Zigachev */
936b843c749SSergey Zigachev struct amdgpu_device *adev = (struct amdgpu_device *)handle;
937b843c749SSergey Zigachev
938b843c749SSergey Zigachev if (!(adev->pg_flags & AMD_PG_SUPPORT_VCE))
939b843c749SSergey Zigachev return 0;
940b843c749SSergey Zigachev
941b843c749SSergey Zigachev if (state == AMD_PG_STATE_GATE)
942b843c749SSergey Zigachev /* XXX do we need a vce_v4_0_stop()? */
943b843c749SSergey Zigachev return 0;
944b843c749SSergey Zigachev else
945b843c749SSergey Zigachev return vce_v4_0_start(adev);
946b843c749SSergey Zigachev }
947b843c749SSergey Zigachev #endif
948b843c749SSergey Zigachev
vce_v4_0_ring_emit_ib(struct amdgpu_ring * ring,struct amdgpu_ib * ib,unsigned int vmid,bool ctx_switch)949b843c749SSergey Zigachev static void vce_v4_0_ring_emit_ib(struct amdgpu_ring *ring,
950b843c749SSergey Zigachev struct amdgpu_ib *ib, unsigned int vmid, bool ctx_switch)
951b843c749SSergey Zigachev {
952b843c749SSergey Zigachev amdgpu_ring_write(ring, VCE_CMD_IB_VM);
953b843c749SSergey Zigachev amdgpu_ring_write(ring, vmid);
954b843c749SSergey Zigachev amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
955b843c749SSergey Zigachev amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
956b843c749SSergey Zigachev amdgpu_ring_write(ring, ib->length_dw);
957b843c749SSergey Zigachev }
958b843c749SSergey Zigachev
vce_v4_0_ring_emit_fence(struct amdgpu_ring * ring,uint64_t addr,uint64_t seq,unsigned flags)959*78973132SSergey Zigachev static void vce_v4_0_ring_emit_fence(struct amdgpu_ring *ring, uint64_t addr,
960*78973132SSergey Zigachev uint64_t seq, unsigned flags)
961b843c749SSergey Zigachev {
962b843c749SSergey Zigachev WARN_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
963b843c749SSergey Zigachev
964b843c749SSergey Zigachev amdgpu_ring_write(ring, VCE_CMD_FENCE);
965b843c749SSergey Zigachev amdgpu_ring_write(ring, addr);
966b843c749SSergey Zigachev amdgpu_ring_write(ring, upper_32_bits(addr));
967b843c749SSergey Zigachev amdgpu_ring_write(ring, seq);
968b843c749SSergey Zigachev amdgpu_ring_write(ring, VCE_CMD_TRAP);
969b843c749SSergey Zigachev }
970b843c749SSergey Zigachev
vce_v4_0_ring_insert_end(struct amdgpu_ring * ring)971b843c749SSergey Zigachev static void vce_v4_0_ring_insert_end(struct amdgpu_ring *ring)
972b843c749SSergey Zigachev {
973b843c749SSergey Zigachev amdgpu_ring_write(ring, VCE_CMD_END);
974b843c749SSergey Zigachev }
975b843c749SSergey Zigachev
vce_v4_0_emit_reg_wait(struct amdgpu_ring * ring,uint32_t reg,uint32_t val,uint32_t mask)976b843c749SSergey Zigachev static void vce_v4_0_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
977b843c749SSergey Zigachev uint32_t val, uint32_t mask)
978b843c749SSergey Zigachev {
979b843c749SSergey Zigachev amdgpu_ring_write(ring, VCE_CMD_REG_WAIT);
980b843c749SSergey Zigachev amdgpu_ring_write(ring, reg << 2);
981b843c749SSergey Zigachev amdgpu_ring_write(ring, mask);
982b843c749SSergey Zigachev amdgpu_ring_write(ring, val);
983b843c749SSergey Zigachev }
984b843c749SSergey Zigachev
vce_v4_0_emit_vm_flush(struct amdgpu_ring * ring,unsigned int vmid,uint64_t pd_addr)985b843c749SSergey Zigachev static void vce_v4_0_emit_vm_flush(struct amdgpu_ring *ring,
986b843c749SSergey Zigachev unsigned int vmid, uint64_t pd_addr)
987b843c749SSergey Zigachev {
988b843c749SSergey Zigachev struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->funcs->vmhub];
989b843c749SSergey Zigachev
990b843c749SSergey Zigachev pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
991b843c749SSergey Zigachev
992b843c749SSergey Zigachev /* wait for reg writes */
993b843c749SSergey Zigachev vce_v4_0_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + vmid * 2,
994b843c749SSergey Zigachev lower_32_bits(pd_addr), 0xffffffff);
995b843c749SSergey Zigachev }
996b843c749SSergey Zigachev
vce_v4_0_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)997b843c749SSergey Zigachev static void vce_v4_0_emit_wreg(struct amdgpu_ring *ring,
998b843c749SSergey Zigachev uint32_t reg, uint32_t val)
999b843c749SSergey Zigachev {
1000b843c749SSergey Zigachev amdgpu_ring_write(ring, VCE_CMD_REG_WRITE);
1001b843c749SSergey Zigachev amdgpu_ring_write(ring, reg << 2);
1002b843c749SSergey Zigachev amdgpu_ring_write(ring, val);
1003b843c749SSergey Zigachev }
1004b843c749SSergey Zigachev
vce_v4_0_set_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)1005b843c749SSergey Zigachev static int vce_v4_0_set_interrupt_state(struct amdgpu_device *adev,
1006b843c749SSergey Zigachev struct amdgpu_irq_src *source,
1007b843c749SSergey Zigachev unsigned type,
1008b843c749SSergey Zigachev enum amdgpu_interrupt_state state)
1009b843c749SSergey Zigachev {
1010b843c749SSergey Zigachev uint32_t val = 0;
1011b843c749SSergey Zigachev
1012b843c749SSergey Zigachev if (!amdgpu_sriov_vf(adev)) {
1013b843c749SSergey Zigachev if (state == AMDGPU_IRQ_STATE_ENABLE)
1014b843c749SSergey Zigachev val |= VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK;
1015b843c749SSergey Zigachev
1016b843c749SSergey Zigachev WREG32_P(SOC15_REG_OFFSET(VCE, 0, mmVCE_SYS_INT_EN), val,
1017b843c749SSergey Zigachev ~VCE_SYS_INT_EN__VCE_SYS_INT_TRAP_INTERRUPT_EN_MASK);
1018b843c749SSergey Zigachev }
1019b843c749SSergey Zigachev return 0;
1020b843c749SSergey Zigachev }
1021b843c749SSergey Zigachev
vce_v4_0_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1022b843c749SSergey Zigachev static int vce_v4_0_process_interrupt(struct amdgpu_device *adev,
1023b843c749SSergey Zigachev struct amdgpu_irq_src *source,
1024b843c749SSergey Zigachev struct amdgpu_iv_entry *entry)
1025b843c749SSergey Zigachev {
1026b843c749SSergey Zigachev DRM_DEBUG("IH: VCE\n");
1027b843c749SSergey Zigachev
1028b843c749SSergey Zigachev switch (entry->src_data[0]) {
1029b843c749SSergey Zigachev case 0:
1030b843c749SSergey Zigachev case 1:
1031b843c749SSergey Zigachev case 2:
1032b843c749SSergey Zigachev amdgpu_fence_process(&adev->vce.ring[entry->src_data[0]]);
1033b843c749SSergey Zigachev break;
1034b843c749SSergey Zigachev default:
1035b843c749SSergey Zigachev DRM_ERROR("Unhandled interrupt: %d %d\n",
1036b843c749SSergey Zigachev entry->src_id, entry->src_data[0]);
1037b843c749SSergey Zigachev break;
1038b843c749SSergey Zigachev }
1039b843c749SSergey Zigachev
1040b843c749SSergey Zigachev return 0;
1041b843c749SSergey Zigachev }
1042b843c749SSergey Zigachev
1043b843c749SSergey Zigachev const struct amd_ip_funcs vce_v4_0_ip_funcs = {
1044b843c749SSergey Zigachev .name = "vce_v4_0",
1045b843c749SSergey Zigachev .early_init = vce_v4_0_early_init,
1046b843c749SSergey Zigachev .late_init = NULL,
1047b843c749SSergey Zigachev .sw_init = vce_v4_0_sw_init,
1048b843c749SSergey Zigachev .sw_fini = vce_v4_0_sw_fini,
1049b843c749SSergey Zigachev .hw_init = vce_v4_0_hw_init,
1050b843c749SSergey Zigachev .hw_fini = vce_v4_0_hw_fini,
1051b843c749SSergey Zigachev .suspend = vce_v4_0_suspend,
1052b843c749SSergey Zigachev .resume = vce_v4_0_resume,
1053b843c749SSergey Zigachev .is_idle = NULL /* vce_v4_0_is_idle */,
1054b843c749SSergey Zigachev .wait_for_idle = NULL /* vce_v4_0_wait_for_idle */,
1055b843c749SSergey Zigachev .check_soft_reset = NULL /* vce_v4_0_check_soft_reset */,
1056b843c749SSergey Zigachev .pre_soft_reset = NULL /* vce_v4_0_pre_soft_reset */,
1057b843c749SSergey Zigachev .soft_reset = NULL /* vce_v4_0_soft_reset */,
1058b843c749SSergey Zigachev .post_soft_reset = NULL /* vce_v4_0_post_soft_reset */,
1059b843c749SSergey Zigachev .set_clockgating_state = vce_v4_0_set_clockgating_state,
1060b843c749SSergey Zigachev .set_powergating_state = NULL /* vce_v4_0_set_powergating_state */,
1061b843c749SSergey Zigachev };
1062b843c749SSergey Zigachev
1063b843c749SSergey Zigachev static const struct amdgpu_ring_funcs vce_v4_0_ring_vm_funcs = {
1064b843c749SSergey Zigachev .type = AMDGPU_RING_TYPE_VCE,
1065b843c749SSergey Zigachev .align_mask = 0x3f,
1066b843c749SSergey Zigachev .nop = VCE_CMD_NO_OP,
1067b843c749SSergey Zigachev .support_64bit_ptrs = false,
1068b843c749SSergey Zigachev .vmhub = AMDGPU_MMHUB,
1069b843c749SSergey Zigachev .get_rptr = vce_v4_0_ring_get_rptr,
1070b843c749SSergey Zigachev .get_wptr = vce_v4_0_ring_get_wptr,
1071b843c749SSergey Zigachev .set_wptr = vce_v4_0_ring_set_wptr,
1072b843c749SSergey Zigachev .parse_cs = amdgpu_vce_ring_parse_cs_vm,
1073b843c749SSergey Zigachev .emit_frame_size =
1074b843c749SSergey Zigachev SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
1075b843c749SSergey Zigachev SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 +
1076b843c749SSergey Zigachev 4 + /* vce_v4_0_emit_vm_flush */
1077b843c749SSergey Zigachev 5 + 5 + /* amdgpu_vce_ring_emit_fence x2 vm fence */
1078b843c749SSergey Zigachev 1, /* vce_v4_0_ring_insert_end */
1079b843c749SSergey Zigachev .emit_ib_size = 5, /* vce_v4_0_ring_emit_ib */
1080b843c749SSergey Zigachev .emit_ib = vce_v4_0_ring_emit_ib,
1081b843c749SSergey Zigachev .emit_vm_flush = vce_v4_0_emit_vm_flush,
1082b843c749SSergey Zigachev .emit_fence = vce_v4_0_ring_emit_fence,
1083b843c749SSergey Zigachev .test_ring = amdgpu_vce_ring_test_ring,
1084b843c749SSergey Zigachev .test_ib = amdgpu_vce_ring_test_ib,
1085b843c749SSergey Zigachev .insert_nop = amdgpu_ring_insert_nop,
1086b843c749SSergey Zigachev .insert_end = vce_v4_0_ring_insert_end,
1087b843c749SSergey Zigachev .pad_ib = amdgpu_ring_generic_pad_ib,
1088b843c749SSergey Zigachev .begin_use = amdgpu_vce_ring_begin_use,
1089b843c749SSergey Zigachev .end_use = amdgpu_vce_ring_end_use,
1090b843c749SSergey Zigachev .emit_wreg = vce_v4_0_emit_wreg,
1091b843c749SSergey Zigachev .emit_reg_wait = vce_v4_0_emit_reg_wait,
1092b843c749SSergey Zigachev .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
1093b843c749SSergey Zigachev };
1094b843c749SSergey Zigachev
vce_v4_0_set_ring_funcs(struct amdgpu_device * adev)1095b843c749SSergey Zigachev static void vce_v4_0_set_ring_funcs(struct amdgpu_device *adev)
1096b843c749SSergey Zigachev {
1097b843c749SSergey Zigachev int i;
1098b843c749SSergey Zigachev
1099b843c749SSergey Zigachev for (i = 0; i < adev->vce.num_rings; i++) {
1100b843c749SSergey Zigachev adev->vce.ring[i].funcs = &vce_v4_0_ring_vm_funcs;
1101b843c749SSergey Zigachev adev->vce.ring[i].me = i;
1102b843c749SSergey Zigachev }
1103b843c749SSergey Zigachev DRM_INFO("VCE enabled in VM mode\n");
1104b843c749SSergey Zigachev }
1105b843c749SSergey Zigachev
1106b843c749SSergey Zigachev static const struct amdgpu_irq_src_funcs vce_v4_0_irq_funcs = {
1107b843c749SSergey Zigachev .set = vce_v4_0_set_interrupt_state,
1108b843c749SSergey Zigachev .process = vce_v4_0_process_interrupt,
1109b843c749SSergey Zigachev };
1110b843c749SSergey Zigachev
vce_v4_0_set_irq_funcs(struct amdgpu_device * adev)1111b843c749SSergey Zigachev static void vce_v4_0_set_irq_funcs(struct amdgpu_device *adev)
1112b843c749SSergey Zigachev {
1113b843c749SSergey Zigachev adev->vce.irq.num_types = 1;
1114b843c749SSergey Zigachev adev->vce.irq.funcs = &vce_v4_0_irq_funcs;
1115b843c749SSergey Zigachev };
1116b843c749SSergey Zigachev
1117b843c749SSergey Zigachev const struct amdgpu_ip_block_version vce_v4_0_ip_block =
1118b843c749SSergey Zigachev {
1119b843c749SSergey Zigachev .type = AMD_IP_BLOCK_TYPE_VCE,
1120b843c749SSergey Zigachev .major = 4,
1121b843c749SSergey Zigachev .minor = 0,
1122b843c749SSergey Zigachev .rev = 0,
1123b843c749SSergey Zigachev .funcs = &vce_v4_0_ip_funcs,
1124b843c749SSergey Zigachev };
1125