1 /*
2  * Copyright 2018 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #ifndef AMDGPU_DOORBELL_H
25 #define AMDGPU_DOORBELL_H
26 
27 /*
28  * GPU doorbell structures, functions & helpers
29  */
30 struct amdgpu_doorbell {
31 	/* doorbell mmio */
32 	resource_size_t		base;
33 	resource_size_t		size;
34 
35 	/* Number of doorbells reserved for amdgpu kernel driver */
36 	u32 num_kernel_doorbells;
37 
38 	/* Kernel doorbells */
39 	struct amdgpu_bo *kernel_doorbells;
40 
41 	/* For CPU access of doorbells */
42 	uint32_t *cpu_addr;
43 };
44 
45 /* Reserved doorbells for amdgpu (including multimedia).
46  * KFD can use all the rest in the 2M doorbell bar.
47  * For asic before vega10, doorbell is 32-bit, so the
48  * index/offset is in dword. For vega10 and after, doorbell
49  * can be 64-bit, so the index defined is in qword.
50  */
51 struct amdgpu_doorbell_index {
52 	uint32_t kiq;
53 	uint32_t mec_ring0;
54 	uint32_t mec_ring1;
55 	uint32_t mec_ring2;
56 	uint32_t mec_ring3;
57 	uint32_t mec_ring4;
58 	uint32_t mec_ring5;
59 	uint32_t mec_ring6;
60 	uint32_t mec_ring7;
61 	uint32_t userqueue_start;
62 	uint32_t userqueue_end;
63 	uint32_t gfx_ring0;
64 	uint32_t gfx_ring1;
65 	uint32_t gfx_userqueue_start;
66 	uint32_t gfx_userqueue_end;
67 	uint32_t sdma_engine[16];
68 	uint32_t mes_ring0;
69 	uint32_t mes_ring1;
70 	uint32_t ih;
71 	union {
72 		struct {
73 			uint32_t vcn_ring0_1;
74 			uint32_t vcn_ring2_3;
75 			uint32_t vcn_ring4_5;
76 			uint32_t vcn_ring6_7;
77 		} vcn;
78 		struct {
79 			uint32_t uvd_ring0_1;
80 			uint32_t uvd_ring2_3;
81 			uint32_t uvd_ring4_5;
82 			uint32_t uvd_ring6_7;
83 			uint32_t vce_ring0_1;
84 			uint32_t vce_ring2_3;
85 			uint32_t vce_ring4_5;
86 			uint32_t vce_ring6_7;
87 		} uvd_vce;
88 	};
89 	uint32_t vpe_ring;
90 	uint32_t first_non_cp;
91 	uint32_t last_non_cp;
92 	uint32_t max_assignment;
93 	/* Per engine SDMA doorbell size in dword */
94 	uint32_t sdma_doorbell_range;
95 	/* Per xcc doorbell size for KIQ/KCQ */
96 	uint32_t xcc_doorbell_range;
97 };
98 
99 enum AMDGPU_DOORBELL_ASSIGNMENT {
100 	AMDGPU_DOORBELL_KIQ                     = 0x000,
101 	AMDGPU_DOORBELL_HIQ                     = 0x001,
102 	AMDGPU_DOORBELL_DIQ                     = 0x002,
103 	AMDGPU_DOORBELL_MEC_RING0               = 0x010,
104 	AMDGPU_DOORBELL_MEC_RING1               = 0x011,
105 	AMDGPU_DOORBELL_MEC_RING2               = 0x012,
106 	AMDGPU_DOORBELL_MEC_RING3               = 0x013,
107 	AMDGPU_DOORBELL_MEC_RING4               = 0x014,
108 	AMDGPU_DOORBELL_MEC_RING5               = 0x015,
109 	AMDGPU_DOORBELL_MEC_RING6               = 0x016,
110 	AMDGPU_DOORBELL_MEC_RING7               = 0x017,
111 	AMDGPU_DOORBELL_GFX_RING0               = 0x020,
112 	AMDGPU_DOORBELL_sDMA_ENGINE0            = 0x1E0,
113 	AMDGPU_DOORBELL_sDMA_ENGINE1            = 0x1E1,
114 	AMDGPU_DOORBELL_IH                      = 0x1E8,
115 	AMDGPU_DOORBELL_MAX_ASSIGNMENT          = 0x3FF,
116 	AMDGPU_DOORBELL_INVALID                 = 0xFFFF
117 };
118 
119 enum AMDGPU_VEGA20_DOORBELL_ASSIGNMENT {
120 
121 	/* Compute + GFX: 0~255 */
122 	AMDGPU_VEGA20_DOORBELL_KIQ                     = 0x000,
123 	AMDGPU_VEGA20_DOORBELL_HIQ                     = 0x001,
124 	AMDGPU_VEGA20_DOORBELL_DIQ                     = 0x002,
125 	AMDGPU_VEGA20_DOORBELL_MEC_RING0               = 0x003,
126 	AMDGPU_VEGA20_DOORBELL_MEC_RING1               = 0x004,
127 	AMDGPU_VEGA20_DOORBELL_MEC_RING2               = 0x005,
128 	AMDGPU_VEGA20_DOORBELL_MEC_RING3               = 0x006,
129 	AMDGPU_VEGA20_DOORBELL_MEC_RING4               = 0x007,
130 	AMDGPU_VEGA20_DOORBELL_MEC_RING5               = 0x008,
131 	AMDGPU_VEGA20_DOORBELL_MEC_RING6               = 0x009,
132 	AMDGPU_VEGA20_DOORBELL_MEC_RING7               = 0x00A,
133 	AMDGPU_VEGA20_DOORBELL_USERQUEUE_START	       = 0x00B,
134 	AMDGPU_VEGA20_DOORBELL_USERQUEUE_END	       = 0x08A,
135 	AMDGPU_VEGA20_DOORBELL_GFX_RING0               = 0x08B,
136 	/* SDMA:256~335*/
137 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0            = 0x100,
138 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE1            = 0x10A,
139 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE2            = 0x114,
140 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE3            = 0x11E,
141 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE4            = 0x128,
142 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE5            = 0x132,
143 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE6            = 0x13C,
144 	AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE7            = 0x146,
145 	/* IH: 376~391 */
146 	AMDGPU_VEGA20_DOORBELL_IH                      = 0x178,
147 	/* MMSCH: 392~407
148 	 * overlap the doorbell assignment with VCN as they are  mutually exclusive
149 	 * VCN engine's doorbell is 32 bit and two VCN ring share one QWORD
150 	 */
151 	AMDGPU_VEGA20_DOORBELL64_VCN0_1                  = 0x188, /* VNC0 */
152 	AMDGPU_VEGA20_DOORBELL64_VCN2_3                  = 0x189,
153 	AMDGPU_VEGA20_DOORBELL64_VCN4_5                  = 0x18A,
154 	AMDGPU_VEGA20_DOORBELL64_VCN6_7                  = 0x18B,
155 
156 	AMDGPU_VEGA20_DOORBELL64_VCN8_9                  = 0x18C, /* VNC1 */
157 	AMDGPU_VEGA20_DOORBELL64_VCNa_b                  = 0x18D,
158 	AMDGPU_VEGA20_DOORBELL64_VCNc_d                  = 0x18E,
159 	AMDGPU_VEGA20_DOORBELL64_VCNe_f                  = 0x18F,
160 
161 	AMDGPU_VEGA20_DOORBELL64_UVD_RING0_1             = 0x188,
162 	AMDGPU_VEGA20_DOORBELL64_UVD_RING2_3             = 0x189,
163 	AMDGPU_VEGA20_DOORBELL64_UVD_RING4_5             = 0x18A,
164 	AMDGPU_VEGA20_DOORBELL64_UVD_RING6_7             = 0x18B,
165 
166 	AMDGPU_VEGA20_DOORBELL64_VCE_RING0_1             = 0x18C,
167 	AMDGPU_VEGA20_DOORBELL64_VCE_RING2_3             = 0x18D,
168 	AMDGPU_VEGA20_DOORBELL64_VCE_RING4_5             = 0x18E,
169 	AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7             = 0x18F,
170 
171 	AMDGPU_VEGA20_DOORBELL64_FIRST_NON_CP            = AMDGPU_VEGA20_DOORBELL_sDMA_ENGINE0,
172 	AMDGPU_VEGA20_DOORBELL64_LAST_NON_CP             = AMDGPU_VEGA20_DOORBELL64_VCE_RING6_7,
173 
174 	/* kiq/kcq from second XCD. Max 8 XCDs */
175 	AMDGPU_VEGA20_DOORBELL_XCC1_KIQ_START             = 0x190,
176 	/* 8 compute rings per GC. Max to 0x1CE */
177 	AMDGPU_VEGA20_DOORBELL_XCC1_MEC_RING0_START       = 0x197,
178 
179 	/* AID1 SDMA: 0x1D0 ~ 0x1F7 */
180 	AMDGPU_VEGA20_DOORBELL_AID1_sDMA_START           = 0x1D0,
181 
182 	AMDGPU_VEGA20_DOORBELL_MAX_ASSIGNMENT            = 0x1F7,
183 	AMDGPU_VEGA20_DOORBELL_INVALID                   = 0xFFFF
184 };
185 
186 enum AMDGPU_NAVI10_DOORBELL_ASSIGNMENT {
187 
188 	/* Compute + GFX: 0~255 */
189 	AMDGPU_NAVI10_DOORBELL_KIQ			= 0x000,
190 	AMDGPU_NAVI10_DOORBELL_HIQ			= 0x001,
191 	AMDGPU_NAVI10_DOORBELL_DIQ			= 0x002,
192 	AMDGPU_NAVI10_DOORBELL_MEC_RING0		= 0x003,
193 	AMDGPU_NAVI10_DOORBELL_MEC_RING1		= 0x004,
194 	AMDGPU_NAVI10_DOORBELL_MEC_RING2		= 0x005,
195 	AMDGPU_NAVI10_DOORBELL_MEC_RING3		= 0x006,
196 	AMDGPU_NAVI10_DOORBELL_MEC_RING4		= 0x007,
197 	AMDGPU_NAVI10_DOORBELL_MEC_RING5		= 0x008,
198 	AMDGPU_NAVI10_DOORBELL_MEC_RING6		= 0x009,
199 	AMDGPU_NAVI10_DOORBELL_MEC_RING7		= 0x00A,
200 	AMDGPU_NAVI10_DOORBELL_MES_RING0	        = 0x00B,
201 	AMDGPU_NAVI10_DOORBELL_MES_RING1		= 0x00C,
202 	AMDGPU_NAVI10_DOORBELL_USERQUEUE_START		= 0x00D,
203 	AMDGPU_NAVI10_DOORBELL_USERQUEUE_END		= 0x08A,
204 	AMDGPU_NAVI10_DOORBELL_GFX_RING0		= 0x08B,
205 	AMDGPU_NAVI10_DOORBELL_GFX_RING1		= 0x08C,
206 	AMDGPU_NAVI10_DOORBELL_GFX_USERQUEUE_START	= 0x08D,
207 	AMDGPU_NAVI10_DOORBELL_GFX_USERQUEUE_END	= 0x0FF,
208 
209 	/* SDMA:256~335*/
210 	AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0		= 0x100,
211 	AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE1		= 0x10A,
212 	AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE2		= 0x114,
213 	AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE3		= 0x11E,
214 	/* IH: 376~391 */
215 	AMDGPU_NAVI10_DOORBELL_IH			= 0x178,
216 	/* MMSCH: 392~407
217 	 * overlap the doorbell assignment with VCN as they are  mutually exclusive
218 	 * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
219 	 */
220 	AMDGPU_NAVI10_DOORBELL64_VCN0_1			= 0x188, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
221 	AMDGPU_NAVI10_DOORBELL64_VCN2_3			= 0x189,
222 	AMDGPU_NAVI10_DOORBELL64_VCN4_5			= 0x18A,
223 	AMDGPU_NAVI10_DOORBELL64_VCN6_7			= 0x18B,
224 
225 	AMDGPU_NAVI10_DOORBELL64_VCN8_9			= 0x18C,
226 	AMDGPU_NAVI10_DOORBELL64_VCNa_b			= 0x18D,
227 	AMDGPU_NAVI10_DOORBELL64_VCNc_d			= 0x18E,
228 	AMDGPU_NAVI10_DOORBELL64_VCNe_f			= 0x18F,
229 
230 	AMDGPU_NAVI10_DOORBELL64_VPE			= 0x190,
231 
232 	AMDGPU_NAVI10_DOORBELL64_FIRST_NON_CP		= AMDGPU_NAVI10_DOORBELL_sDMA_ENGINE0,
233 	AMDGPU_NAVI10_DOORBELL64_LAST_NON_CP		= AMDGPU_NAVI10_DOORBELL64_VPE,
234 
235 	AMDGPU_NAVI10_DOORBELL_MAX_ASSIGNMENT		= AMDGPU_NAVI10_DOORBELL64_VPE,
236 	AMDGPU_NAVI10_DOORBELL_INVALID			= 0xFFFF
237 };
238 
239 /*
240  * 64bit doorbell, offset are in QWORD, occupy 2KB doorbell space
241  */
242 enum AMDGPU_DOORBELL64_ASSIGNMENT {
243 	/*
244 	 * All compute related doorbells: kiq, hiq, diq, traditional compute queue, user queue, should locate in
245 	 * a continues range so that programming CP_MEC_DOORBELL_RANGE_LOWER/UPPER can cover this range.
246 	 *  Compute related doorbells are allocated from 0x00 to 0x8a
247 	 */
248 
249 
250 	/* kernel scheduling */
251 	AMDGPU_DOORBELL64_KIQ                     = 0x00,
252 
253 	/* HSA interface queue and debug queue */
254 	AMDGPU_DOORBELL64_HIQ                     = 0x01,
255 	AMDGPU_DOORBELL64_DIQ                     = 0x02,
256 
257 	/* Compute engines */
258 	AMDGPU_DOORBELL64_MEC_RING0               = 0x03,
259 	AMDGPU_DOORBELL64_MEC_RING1               = 0x04,
260 	AMDGPU_DOORBELL64_MEC_RING2               = 0x05,
261 	AMDGPU_DOORBELL64_MEC_RING3               = 0x06,
262 	AMDGPU_DOORBELL64_MEC_RING4               = 0x07,
263 	AMDGPU_DOORBELL64_MEC_RING5               = 0x08,
264 	AMDGPU_DOORBELL64_MEC_RING6               = 0x09,
265 	AMDGPU_DOORBELL64_MEC_RING7               = 0x0a,
266 
267 	/* User queue doorbell range (128 doorbells) */
268 	AMDGPU_DOORBELL64_USERQUEUE_START         = 0x0b,
269 	AMDGPU_DOORBELL64_USERQUEUE_END           = 0x8a,
270 
271 	/* Graphics engine */
272 	AMDGPU_DOORBELL64_GFX_RING0               = 0x8b,
273 
274 	/*
275 	 * Other graphics doorbells can be allocated here: from 0x8c to 0xdf
276 	 * Graphics voltage island aperture 1
277 	 * default non-graphics QWORD index is 0xe0 - 0xFF inclusive
278 	 */
279 
280 	/* For vega10 sriov, the sdma doorbell must be fixed as follow
281 	 * to keep the same setting with host driver, or it will
282 	 * happen conflicts
283 	 */
284 	AMDGPU_DOORBELL64_sDMA_ENGINE0            = 0xF0,
285 	AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE0     = 0xF1,
286 	AMDGPU_DOORBELL64_sDMA_ENGINE1            = 0xF2,
287 	AMDGPU_DOORBELL64_sDMA_HI_PRI_ENGINE1     = 0xF3,
288 
289 	/* Interrupt handler */
290 	AMDGPU_DOORBELL64_IH                      = 0xF4,  /* For legacy interrupt ring buffer */
291 	AMDGPU_DOORBELL64_IH_RING1                = 0xF5,  /* For page migration request log */
292 	AMDGPU_DOORBELL64_IH_RING2                = 0xF6,  /* For page migration translation/invalidation log */
293 
294 	/* VCN engine use 32 bits doorbell  */
295 	AMDGPU_DOORBELL64_VCN0_1                  = 0xF8, /* lower 32 bits for VNC0 and upper 32 bits for VNC1 */
296 	AMDGPU_DOORBELL64_VCN2_3                  = 0xF9,
297 	AMDGPU_DOORBELL64_VCN4_5                  = 0xFA,
298 	AMDGPU_DOORBELL64_VCN6_7                  = 0xFB,
299 
300 	/* overlap the doorbell assignment with VCN as they are  mutually exclusive
301 	 * VCE engine's doorbell is 32 bit and two VCE ring share one QWORD
302 	 */
303 	AMDGPU_DOORBELL64_UVD_RING0_1             = 0xF8,
304 	AMDGPU_DOORBELL64_UVD_RING2_3             = 0xF9,
305 	AMDGPU_DOORBELL64_UVD_RING4_5             = 0xFA,
306 	AMDGPU_DOORBELL64_UVD_RING6_7             = 0xFB,
307 
308 	AMDGPU_DOORBELL64_VCE_RING0_1             = 0xFC,
309 	AMDGPU_DOORBELL64_VCE_RING2_3             = 0xFD,
310 	AMDGPU_DOORBELL64_VCE_RING4_5             = 0xFE,
311 	AMDGPU_DOORBELL64_VCE_RING6_7             = 0xFF,
312 
313 	AMDGPU_DOORBELL64_FIRST_NON_CP            = AMDGPU_DOORBELL64_sDMA_ENGINE0,
314 	AMDGPU_DOORBELL64_LAST_NON_CP             = AMDGPU_DOORBELL64_VCE_RING6_7,
315 
316 	AMDGPU_DOORBELL64_MAX_ASSIGNMENT          = 0xFF,
317 	AMDGPU_DOORBELL64_INVALID                 = 0xFFFF
318 };
319 
320 enum AMDGPU_DOORBELL_ASSIGNMENT_LAYOUT1 {
321 
322 	/* XCC0: 0x00 ~20, XCC1: 20 ~ 2F ... */
323 
324 	/* KIQ/HIQ/DIQ */
325 	AMDGPU_DOORBELL_LAYOUT1_KIQ_START		= 0x000,
326 	AMDGPU_DOORBELL_LAYOUT1_HIQ			= 0x001,
327 	AMDGPU_DOORBELL_LAYOUT1_DIQ			= 0x002,
328 	/* Compute: 0x08 ~ 0x20  */
329 	AMDGPU_DOORBELL_LAYOUT1_MEC_RING_START		= 0x008,
330 	AMDGPU_DOORBELL_LAYOUT1_MEC_RING_END		= 0x00F,
331 	AMDGPU_DOORBELL_LAYOUT1_USERQUEUE_START		= 0x010,
332 	AMDGPU_DOORBELL_LAYOUT1_USERQUEUE_END		= 0x01F,
333 	AMDGPU_DOORBELL_LAYOUT1_XCC_RANGE		= 0x020,
334 
335 	/* SDMA: 0x100 ~ 0x19F */
336 	AMDGPU_DOORBELL_LAYOUT1_sDMA_ENGINE_START	= 0x100,
337 	AMDGPU_DOORBELL_LAYOUT1_sDMA_ENGINE_END		= 0x19F,
338 	/* IH: 0x1A0 ~ 0x1AF */
339 	AMDGPU_DOORBELL_LAYOUT1_IH                      = 0x1A0,
340 	/* VCN: 0x1B0 ~ 0x1E8 */
341 	AMDGPU_DOORBELL_LAYOUT1_VCN_START               = 0x1B0,
342 	AMDGPU_DOORBELL_LAYOUT1_VCN_END                 = 0x1E8,
343 
344 	AMDGPU_DOORBELL_LAYOUT1_FIRST_NON_CP		= AMDGPU_DOORBELL_LAYOUT1_sDMA_ENGINE_START,
345 	AMDGPU_DOORBELL_LAYOUT1_LAST_NON_CP		= AMDGPU_DOORBELL_LAYOUT1_VCN_END,
346 
347 	AMDGPU_DOORBELL_LAYOUT1_MAX_ASSIGNMENT          = 0x1E8,
348 	AMDGPU_DOORBELL_LAYOUT1_INVALID                 = 0xFFFF
349 };
350 
351 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index);
352 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
353 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index);
354 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v);
355 
356 /*
357  * GPU doorbell aperture helpers function.
358  */
359 int amdgpu_doorbell_init(struct amdgpu_device *adev);
360 void amdgpu_doorbell_fini(struct amdgpu_device *adev);
361 int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev);
362 uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev,
363 				      struct amdgpu_bo *db_bo,
364 				      uint32_t doorbell_index,
365 				      uint32_t db_size);
366 
367 #define RDOORBELL32(index) amdgpu_mm_rdoorbell(adev, (index))
368 #define WDOORBELL32(index, v) amdgpu_mm_wdoorbell(adev, (index), (v))
369 #define RDOORBELL64(index) amdgpu_mm_rdoorbell64(adev, (index))
370 #define WDOORBELL64(index, v) amdgpu_mm_wdoorbell64(adev, (index), (v))
371 
372 #endif
373