1fb4d8502Sjsg /*
2fb4d8502Sjsg * Copyright 2016 Advanced Micro Devices, Inc.
3fb4d8502Sjsg *
4fb4d8502Sjsg * Permission is hereby granted, free of charge, to any person obtaining a
5fb4d8502Sjsg * copy of this software and associated documentation files (the "Software"),
6fb4d8502Sjsg * to deal in the Software without restriction, including without limitation
7fb4d8502Sjsg * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8fb4d8502Sjsg * and/or sell copies of the Software, and to permit persons to whom the
9fb4d8502Sjsg * Software is furnished to do so, subject to the following conditions:
10fb4d8502Sjsg *
11fb4d8502Sjsg * The above copyright notice and this permission notice shall be included in
12fb4d8502Sjsg * all copies or substantial portions of the Software.
13fb4d8502Sjsg *
14fb4d8502Sjsg * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15fb4d8502Sjsg * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16fb4d8502Sjsg * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17fb4d8502Sjsg * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18fb4d8502Sjsg * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19fb4d8502Sjsg * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20fb4d8502Sjsg * OTHER DEALINGS IN THE SOFTWARE.
21fb4d8502Sjsg *
22fb4d8502Sjsg */
23c349dbc7Sjsg
24fb4d8502Sjsg #include <linux/firmware.h>
25c349dbc7Sjsg #include <linux/pci.h>
26c349dbc7Sjsg
27fb4d8502Sjsg #include <drm/drm_cache.h>
28c349dbc7Sjsg
29fb4d8502Sjsg #include "amdgpu.h"
30fb4d8502Sjsg #include "gmc_v9_0.h"
31fb4d8502Sjsg #include "amdgpu_atomfirmware.h"
32c349dbc7Sjsg #include "amdgpu_gem.h"
33fb4d8502Sjsg
34fb4d8502Sjsg #include "gc/gc_9_0_sh_mask.h"
35fb4d8502Sjsg #include "dce/dce_12_0_offset.h"
36fb4d8502Sjsg #include "dce/dce_12_0_sh_mask.h"
37fb4d8502Sjsg #include "vega10_enum.h"
38fb4d8502Sjsg #include "mmhub/mmhub_1_0_offset.h"
39c349dbc7Sjsg #include "athub/athub_1_0_sh_mask.h"
40fb4d8502Sjsg #include "athub/athub_1_0_offset.h"
41fb4d8502Sjsg #include "oss/osssys_4_0_offset.h"
42fb4d8502Sjsg
43fb4d8502Sjsg #include "soc15.h"
44c349dbc7Sjsg #include "soc15d.h"
45fb4d8502Sjsg #include "soc15_common.h"
46fb4d8502Sjsg #include "umc/umc_6_0_sh_mask.h"
47fb4d8502Sjsg
48fb4d8502Sjsg #include "gfxhub_v1_0.h"
49fb4d8502Sjsg #include "mmhub_v1_0.h"
50c349dbc7Sjsg #include "athub_v1_0.h"
51c349dbc7Sjsg #include "gfxhub_v1_1.h"
52f005ef32Sjsg #include "gfxhub_v1_2.h"
53c349dbc7Sjsg #include "mmhub_v9_4.h"
545ca02815Sjsg #include "mmhub_v1_7.h"
55f005ef32Sjsg #include "mmhub_v1_8.h"
56c349dbc7Sjsg #include "umc_v6_1.h"
57c349dbc7Sjsg #include "umc_v6_0.h"
585ca02815Sjsg #include "umc_v6_7.h"
595ca02815Sjsg #include "hdp_v4_0.h"
605ca02815Sjsg #include "mca_v3_0.h"
61fb4d8502Sjsg
62fb4d8502Sjsg #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
63fb4d8502Sjsg
64c349dbc7Sjsg #include "amdgpu_ras.h"
65c349dbc7Sjsg #include "amdgpu_xgmi.h"
66c349dbc7Sjsg
671bb76ff1Sjsg #include "amdgpu_reset.h"
681bb76ff1Sjsg
69fb4d8502Sjsg /* add these here since we already include dce12 headers and these are for DCN */
70fb4d8502Sjsg #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d
71fb4d8502Sjsg #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
72fb4d8502Sjsg #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0
73fb4d8502Sjsg #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10
74fb4d8502Sjsg #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL
75fb4d8502Sjsg #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L
76ad8b1aafSjsg #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d
77ad8b1aafSjsg #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2
78fb4d8502Sjsg
79c097090cSjsg #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2 0x05ea
80c097090cSjsg #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX 2
81c097090cSjsg
82f005ef32Sjsg #define MAX_MEM_RANGES 8
83ad8b1aafSjsg
84f005ef32Sjsg static const char * const gfxhub_client_ids[] = {
85ad8b1aafSjsg "CB",
86ad8b1aafSjsg "DB",
87ad8b1aafSjsg "IA",
88ad8b1aafSjsg "WD",
89ad8b1aafSjsg "CPF",
90ad8b1aafSjsg "CPC",
91ad8b1aafSjsg "CPG",
92ad8b1aafSjsg "RLC",
93ad8b1aafSjsg "TCP",
94ad8b1aafSjsg "SQC (inst)",
95ad8b1aafSjsg "SQC (data)",
96ad8b1aafSjsg "SQG",
97ad8b1aafSjsg "PA",
98ad8b1aafSjsg };
99ad8b1aafSjsg
100ad8b1aafSjsg static const char *mmhub_client_ids_raven[][2] = {
101ad8b1aafSjsg [0][0] = "MP1",
102ad8b1aafSjsg [1][0] = "MP0",
103ad8b1aafSjsg [2][0] = "VCN",
104ad8b1aafSjsg [3][0] = "VCNU",
105ad8b1aafSjsg [4][0] = "HDP",
106ad8b1aafSjsg [5][0] = "DCE",
107ad8b1aafSjsg [13][0] = "UTCL2",
108ad8b1aafSjsg [19][0] = "TLS",
109ad8b1aafSjsg [26][0] = "OSS",
110ad8b1aafSjsg [27][0] = "SDMA0",
111ad8b1aafSjsg [0][1] = "MP1",
112ad8b1aafSjsg [1][1] = "MP0",
113ad8b1aafSjsg [2][1] = "VCN",
114ad8b1aafSjsg [3][1] = "VCNU",
115ad8b1aafSjsg [4][1] = "HDP",
116ad8b1aafSjsg [5][1] = "XDP",
117ad8b1aafSjsg [6][1] = "DBGU0",
118ad8b1aafSjsg [7][1] = "DCE",
119ad8b1aafSjsg [8][1] = "DCEDWB0",
120ad8b1aafSjsg [9][1] = "DCEDWB1",
121ad8b1aafSjsg [26][1] = "OSS",
122ad8b1aafSjsg [27][1] = "SDMA0",
123ad8b1aafSjsg };
124ad8b1aafSjsg
125ad8b1aafSjsg static const char *mmhub_client_ids_renoir[][2] = {
126ad8b1aafSjsg [0][0] = "MP1",
127ad8b1aafSjsg [1][0] = "MP0",
128ad8b1aafSjsg [2][0] = "HDP",
129ad8b1aafSjsg [4][0] = "DCEDMC",
130ad8b1aafSjsg [5][0] = "DCEVGA",
131ad8b1aafSjsg [13][0] = "UTCL2",
132ad8b1aafSjsg [19][0] = "TLS",
133ad8b1aafSjsg [26][0] = "OSS",
134ad8b1aafSjsg [27][0] = "SDMA0",
135ad8b1aafSjsg [28][0] = "VCN",
136ad8b1aafSjsg [29][0] = "VCNU",
137ad8b1aafSjsg [30][0] = "JPEG",
138ad8b1aafSjsg [0][1] = "MP1",
139ad8b1aafSjsg [1][1] = "MP0",
140ad8b1aafSjsg [2][1] = "HDP",
141ad8b1aafSjsg [3][1] = "XDP",
142ad8b1aafSjsg [6][1] = "DBGU0",
143ad8b1aafSjsg [7][1] = "DCEDMC",
144ad8b1aafSjsg [8][1] = "DCEVGA",
145ad8b1aafSjsg [9][1] = "DCEDWB",
146ad8b1aafSjsg [26][1] = "OSS",
147ad8b1aafSjsg [27][1] = "SDMA0",
148ad8b1aafSjsg [28][1] = "VCN",
149ad8b1aafSjsg [29][1] = "VCNU",
150ad8b1aafSjsg [30][1] = "JPEG",
151ad8b1aafSjsg };
152ad8b1aafSjsg
153ad8b1aafSjsg static const char *mmhub_client_ids_vega10[][2] = {
154ad8b1aafSjsg [0][0] = "MP0",
155ad8b1aafSjsg [1][0] = "UVD",
156ad8b1aafSjsg [2][0] = "UVDU",
157ad8b1aafSjsg [3][0] = "HDP",
158ad8b1aafSjsg [13][0] = "UTCL2",
159ad8b1aafSjsg [14][0] = "OSS",
160ad8b1aafSjsg [15][0] = "SDMA1",
161ad8b1aafSjsg [32+0][0] = "VCE0",
162ad8b1aafSjsg [32+1][0] = "VCE0U",
163ad8b1aafSjsg [32+2][0] = "XDMA",
164ad8b1aafSjsg [32+3][0] = "DCE",
165ad8b1aafSjsg [32+4][0] = "MP1",
166ad8b1aafSjsg [32+14][0] = "SDMA0",
167ad8b1aafSjsg [0][1] = "MP0",
168ad8b1aafSjsg [1][1] = "UVD",
169ad8b1aafSjsg [2][1] = "UVDU",
170ad8b1aafSjsg [3][1] = "DBGU0",
171ad8b1aafSjsg [4][1] = "HDP",
172ad8b1aafSjsg [5][1] = "XDP",
173ad8b1aafSjsg [14][1] = "OSS",
174ad8b1aafSjsg [15][1] = "SDMA0",
175ad8b1aafSjsg [32+0][1] = "VCE0",
176ad8b1aafSjsg [32+1][1] = "VCE0U",
177ad8b1aafSjsg [32+2][1] = "XDMA",
178ad8b1aafSjsg [32+3][1] = "DCE",
179ad8b1aafSjsg [32+4][1] = "DCEDWB",
180ad8b1aafSjsg [32+5][1] = "MP1",
181ad8b1aafSjsg [32+6][1] = "DBGU1",
182ad8b1aafSjsg [32+14][1] = "SDMA1",
183ad8b1aafSjsg };
184ad8b1aafSjsg
185ad8b1aafSjsg static const char *mmhub_client_ids_vega12[][2] = {
186ad8b1aafSjsg [0][0] = "MP0",
187ad8b1aafSjsg [1][0] = "VCE0",
188ad8b1aafSjsg [2][0] = "VCE0U",
189ad8b1aafSjsg [3][0] = "HDP",
190ad8b1aafSjsg [13][0] = "UTCL2",
191ad8b1aafSjsg [14][0] = "OSS",
192ad8b1aafSjsg [15][0] = "SDMA1",
193ad8b1aafSjsg [32+0][0] = "DCE",
194ad8b1aafSjsg [32+1][0] = "XDMA",
195ad8b1aafSjsg [32+2][0] = "UVD",
196ad8b1aafSjsg [32+3][0] = "UVDU",
197ad8b1aafSjsg [32+4][0] = "MP1",
198ad8b1aafSjsg [32+15][0] = "SDMA0",
199ad8b1aafSjsg [0][1] = "MP0",
200ad8b1aafSjsg [1][1] = "VCE0",
201ad8b1aafSjsg [2][1] = "VCE0U",
202ad8b1aafSjsg [3][1] = "DBGU0",
203ad8b1aafSjsg [4][1] = "HDP",
204ad8b1aafSjsg [5][1] = "XDP",
205ad8b1aafSjsg [14][1] = "OSS",
206ad8b1aafSjsg [15][1] = "SDMA0",
207ad8b1aafSjsg [32+0][1] = "DCE",
208ad8b1aafSjsg [32+1][1] = "DCEDWB",
209ad8b1aafSjsg [32+2][1] = "XDMA",
210ad8b1aafSjsg [32+3][1] = "UVD",
211ad8b1aafSjsg [32+4][1] = "UVDU",
212ad8b1aafSjsg [32+5][1] = "MP1",
213ad8b1aafSjsg [32+6][1] = "DBGU1",
214ad8b1aafSjsg [32+15][1] = "SDMA1",
215ad8b1aafSjsg };
216ad8b1aafSjsg
217ad8b1aafSjsg static const char *mmhub_client_ids_vega20[][2] = {
218ad8b1aafSjsg [0][0] = "XDMA",
219ad8b1aafSjsg [1][0] = "DCE",
220ad8b1aafSjsg [2][0] = "VCE0",
221ad8b1aafSjsg [3][0] = "VCE0U",
222ad8b1aafSjsg [4][0] = "UVD",
223ad8b1aafSjsg [5][0] = "UVD1U",
224ad8b1aafSjsg [13][0] = "OSS",
225ad8b1aafSjsg [14][0] = "HDP",
226ad8b1aafSjsg [15][0] = "SDMA0",
227ad8b1aafSjsg [32+0][0] = "UVD",
228ad8b1aafSjsg [32+1][0] = "UVDU",
229ad8b1aafSjsg [32+2][0] = "MP1",
230ad8b1aafSjsg [32+3][0] = "MP0",
231ad8b1aafSjsg [32+12][0] = "UTCL2",
232ad8b1aafSjsg [32+14][0] = "SDMA1",
233ad8b1aafSjsg [0][1] = "XDMA",
234ad8b1aafSjsg [1][1] = "DCE",
235ad8b1aafSjsg [2][1] = "DCEDWB",
236ad8b1aafSjsg [3][1] = "VCE0",
237ad8b1aafSjsg [4][1] = "VCE0U",
238ad8b1aafSjsg [5][1] = "UVD1",
239ad8b1aafSjsg [6][1] = "UVD1U",
240ad8b1aafSjsg [7][1] = "DBGU0",
241ad8b1aafSjsg [8][1] = "XDP",
242ad8b1aafSjsg [13][1] = "OSS",
243ad8b1aafSjsg [14][1] = "HDP",
244ad8b1aafSjsg [15][1] = "SDMA0",
245ad8b1aafSjsg [32+0][1] = "UVD",
246ad8b1aafSjsg [32+1][1] = "UVDU",
247ad8b1aafSjsg [32+2][1] = "DBGU1",
248ad8b1aafSjsg [32+3][1] = "MP1",
249ad8b1aafSjsg [32+4][1] = "MP0",
250ad8b1aafSjsg [32+14][1] = "SDMA1",
251ad8b1aafSjsg };
252ad8b1aafSjsg
253ad8b1aafSjsg static const char *mmhub_client_ids_arcturus[][2] = {
2545ca02815Sjsg [0][0] = "DBGU1",
2555ca02815Sjsg [1][0] = "XDP",
256ad8b1aafSjsg [2][0] = "MP1",
257ad8b1aafSjsg [14][0] = "HDP",
2585ca02815Sjsg [171][0] = "JPEG",
2595ca02815Sjsg [172][0] = "VCN",
2605ca02815Sjsg [173][0] = "VCNU",
2615ca02815Sjsg [203][0] = "JPEG1",
2625ca02815Sjsg [204][0] = "VCN1",
2635ca02815Sjsg [205][0] = "VCN1U",
2645ca02815Sjsg [256][0] = "SDMA0",
2655ca02815Sjsg [257][0] = "SDMA1",
2665ca02815Sjsg [258][0] = "SDMA2",
2675ca02815Sjsg [259][0] = "SDMA3",
2685ca02815Sjsg [260][0] = "SDMA4",
2695ca02815Sjsg [261][0] = "SDMA5",
2705ca02815Sjsg [262][0] = "SDMA6",
2715ca02815Sjsg [263][0] = "SDMA7",
2725ca02815Sjsg [384][0] = "OSS",
273ad8b1aafSjsg [0][1] = "DBGU1",
274ad8b1aafSjsg [1][1] = "XDP",
275ad8b1aafSjsg [2][1] = "MP1",
276ad8b1aafSjsg [14][1] = "HDP",
2775ca02815Sjsg [171][1] = "JPEG",
2785ca02815Sjsg [172][1] = "VCN",
2795ca02815Sjsg [173][1] = "VCNU",
2805ca02815Sjsg [203][1] = "JPEG1",
2815ca02815Sjsg [204][1] = "VCN1",
2825ca02815Sjsg [205][1] = "VCN1U",
2835ca02815Sjsg [256][1] = "SDMA0",
2845ca02815Sjsg [257][1] = "SDMA1",
2855ca02815Sjsg [258][1] = "SDMA2",
2865ca02815Sjsg [259][1] = "SDMA3",
2875ca02815Sjsg [260][1] = "SDMA4",
2885ca02815Sjsg [261][1] = "SDMA5",
2895ca02815Sjsg [262][1] = "SDMA6",
2905ca02815Sjsg [263][1] = "SDMA7",
2915ca02815Sjsg [384][1] = "OSS",
292ad8b1aafSjsg };
293fb4d8502Sjsg
2945ca02815Sjsg static const char *mmhub_client_ids_aldebaran[][2] = {
2955ca02815Sjsg [2][0] = "MP1",
2965ca02815Sjsg [3][0] = "MP0",
2975ca02815Sjsg [32+1][0] = "DBGU_IO0",
2985ca02815Sjsg [32+2][0] = "DBGU_IO2",
2995ca02815Sjsg [32+4][0] = "MPIO",
3005ca02815Sjsg [96+11][0] = "JPEG0",
3015ca02815Sjsg [96+12][0] = "VCN0",
3025ca02815Sjsg [96+13][0] = "VCNU0",
3035ca02815Sjsg [128+11][0] = "JPEG1",
3045ca02815Sjsg [128+12][0] = "VCN1",
3055ca02815Sjsg [128+13][0] = "VCNU1",
3065ca02815Sjsg [160+1][0] = "XDP",
3075ca02815Sjsg [160+14][0] = "HDP",
3085ca02815Sjsg [256+0][0] = "SDMA0",
3095ca02815Sjsg [256+1][0] = "SDMA1",
3105ca02815Sjsg [256+2][0] = "SDMA2",
3115ca02815Sjsg [256+3][0] = "SDMA3",
3125ca02815Sjsg [256+4][0] = "SDMA4",
3135ca02815Sjsg [384+0][0] = "OSS",
3145ca02815Sjsg [2][1] = "MP1",
3155ca02815Sjsg [3][1] = "MP0",
3165ca02815Sjsg [32+1][1] = "DBGU_IO0",
3175ca02815Sjsg [32+2][1] = "DBGU_IO2",
3185ca02815Sjsg [32+4][1] = "MPIO",
3195ca02815Sjsg [96+11][1] = "JPEG0",
3205ca02815Sjsg [96+12][1] = "VCN0",
3215ca02815Sjsg [96+13][1] = "VCNU0",
3225ca02815Sjsg [128+11][1] = "JPEG1",
3235ca02815Sjsg [128+12][1] = "VCN1",
3245ca02815Sjsg [128+13][1] = "VCNU1",
3255ca02815Sjsg [160+1][1] = "XDP",
3265ca02815Sjsg [160+14][1] = "HDP",
3275ca02815Sjsg [256+0][1] = "SDMA0",
3285ca02815Sjsg [256+1][1] = "SDMA1",
3295ca02815Sjsg [256+2][1] = "SDMA2",
3305ca02815Sjsg [256+3][1] = "SDMA3",
3315ca02815Sjsg [256+4][1] = "SDMA4",
3325ca02815Sjsg [384+0][1] = "OSS",
333fb4d8502Sjsg };
334fb4d8502Sjsg
335f005ef32Sjsg static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = {
336fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
337fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
338fb4d8502Sjsg };
339fb4d8502Sjsg
340f005ef32Sjsg static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = {
341fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
342fb4d8502Sjsg SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
343fb4d8502Sjsg };
344fb4d8502Sjsg
345c349dbc7Sjsg static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
346c349dbc7Sjsg (0x000143c0 + 0x00000000),
347c349dbc7Sjsg (0x000143c0 + 0x00000800),
348c349dbc7Sjsg (0x000143c0 + 0x00001000),
349c349dbc7Sjsg (0x000143c0 + 0x00001800),
350c349dbc7Sjsg (0x000543c0 + 0x00000000),
351c349dbc7Sjsg (0x000543c0 + 0x00000800),
352c349dbc7Sjsg (0x000543c0 + 0x00001000),
353c349dbc7Sjsg (0x000543c0 + 0x00001800),
354c349dbc7Sjsg (0x000943c0 + 0x00000000),
355c349dbc7Sjsg (0x000943c0 + 0x00000800),
356c349dbc7Sjsg (0x000943c0 + 0x00001000),
357c349dbc7Sjsg (0x000943c0 + 0x00001800),
358c349dbc7Sjsg (0x000d43c0 + 0x00000000),
359c349dbc7Sjsg (0x000d43c0 + 0x00000800),
360c349dbc7Sjsg (0x000d43c0 + 0x00001000),
361c349dbc7Sjsg (0x000d43c0 + 0x00001800),
362c349dbc7Sjsg (0x001143c0 + 0x00000000),
363c349dbc7Sjsg (0x001143c0 + 0x00000800),
364c349dbc7Sjsg (0x001143c0 + 0x00001000),
365c349dbc7Sjsg (0x001143c0 + 0x00001800),
366c349dbc7Sjsg (0x001543c0 + 0x00000000),
367c349dbc7Sjsg (0x001543c0 + 0x00000800),
368c349dbc7Sjsg (0x001543c0 + 0x00001000),
369c349dbc7Sjsg (0x001543c0 + 0x00001800),
370c349dbc7Sjsg (0x001943c0 + 0x00000000),
371c349dbc7Sjsg (0x001943c0 + 0x00000800),
372c349dbc7Sjsg (0x001943c0 + 0x00001000),
373c349dbc7Sjsg (0x001943c0 + 0x00001800),
374c349dbc7Sjsg (0x001d43c0 + 0x00000000),
375c349dbc7Sjsg (0x001d43c0 + 0x00000800),
376c349dbc7Sjsg (0x001d43c0 + 0x00001000),
377c349dbc7Sjsg (0x001d43c0 + 0x00001800),
378fb4d8502Sjsg };
379fb4d8502Sjsg
380c349dbc7Sjsg static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
381c349dbc7Sjsg (0x000143e0 + 0x00000000),
382c349dbc7Sjsg (0x000143e0 + 0x00000800),
383c349dbc7Sjsg (0x000143e0 + 0x00001000),
384c349dbc7Sjsg (0x000143e0 + 0x00001800),
385c349dbc7Sjsg (0x000543e0 + 0x00000000),
386c349dbc7Sjsg (0x000543e0 + 0x00000800),
387c349dbc7Sjsg (0x000543e0 + 0x00001000),
388c349dbc7Sjsg (0x000543e0 + 0x00001800),
389c349dbc7Sjsg (0x000943e0 + 0x00000000),
390c349dbc7Sjsg (0x000943e0 + 0x00000800),
391c349dbc7Sjsg (0x000943e0 + 0x00001000),
392c349dbc7Sjsg (0x000943e0 + 0x00001800),
393c349dbc7Sjsg (0x000d43e0 + 0x00000000),
394c349dbc7Sjsg (0x000d43e0 + 0x00000800),
395c349dbc7Sjsg (0x000d43e0 + 0x00001000),
396c349dbc7Sjsg (0x000d43e0 + 0x00001800),
397c349dbc7Sjsg (0x001143e0 + 0x00000000),
398c349dbc7Sjsg (0x001143e0 + 0x00000800),
399c349dbc7Sjsg (0x001143e0 + 0x00001000),
400c349dbc7Sjsg (0x001143e0 + 0x00001800),
401c349dbc7Sjsg (0x001543e0 + 0x00000000),
402c349dbc7Sjsg (0x001543e0 + 0x00000800),
403c349dbc7Sjsg (0x001543e0 + 0x00001000),
404c349dbc7Sjsg (0x001543e0 + 0x00001800),
405c349dbc7Sjsg (0x001943e0 + 0x00000000),
406c349dbc7Sjsg (0x001943e0 + 0x00000800),
407c349dbc7Sjsg (0x001943e0 + 0x00001000),
408c349dbc7Sjsg (0x001943e0 + 0x00001800),
409c349dbc7Sjsg (0x001d43e0 + 0x00000000),
410c349dbc7Sjsg (0x001d43e0 + 0x00000800),
411c349dbc7Sjsg (0x001d43e0 + 0x00001000),
412c349dbc7Sjsg (0x001d43e0 + 0x00001800),
413fb4d8502Sjsg };
414fb4d8502Sjsg
gmc_v9_0_ecc_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned int type,enum amdgpu_interrupt_state state)415c349dbc7Sjsg static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
416c349dbc7Sjsg struct amdgpu_irq_src *src,
417f005ef32Sjsg unsigned int type,
418c349dbc7Sjsg enum amdgpu_interrupt_state state)
419c349dbc7Sjsg {
420c349dbc7Sjsg u32 bits, i, tmp, reg;
421c349dbc7Sjsg
422c349dbc7Sjsg /* Devices newer then VEGA10/12 shall have these programming
423f005ef32Sjsg * sequences performed by PSP BL
424f005ef32Sjsg */
425c349dbc7Sjsg if (adev->asic_type >= CHIP_VEGA20)
426c349dbc7Sjsg return 0;
427c349dbc7Sjsg
428c349dbc7Sjsg bits = 0x7f;
429c349dbc7Sjsg
430c349dbc7Sjsg switch (state) {
431c349dbc7Sjsg case AMDGPU_IRQ_STATE_DISABLE:
432c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
433c349dbc7Sjsg reg = ecc_umc_mcumc_ctrl_addrs[i];
434c349dbc7Sjsg tmp = RREG32(reg);
435c349dbc7Sjsg tmp &= ~bits;
436c349dbc7Sjsg WREG32(reg, tmp);
437c349dbc7Sjsg }
438c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
439c349dbc7Sjsg reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
440c349dbc7Sjsg tmp = RREG32(reg);
441c349dbc7Sjsg tmp &= ~bits;
442c349dbc7Sjsg WREG32(reg, tmp);
443c349dbc7Sjsg }
444c349dbc7Sjsg break;
445c349dbc7Sjsg case AMDGPU_IRQ_STATE_ENABLE:
446c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
447c349dbc7Sjsg reg = ecc_umc_mcumc_ctrl_addrs[i];
448c349dbc7Sjsg tmp = RREG32(reg);
449c349dbc7Sjsg tmp |= bits;
450c349dbc7Sjsg WREG32(reg, tmp);
451c349dbc7Sjsg }
452c349dbc7Sjsg for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
453c349dbc7Sjsg reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
454c349dbc7Sjsg tmp = RREG32(reg);
455c349dbc7Sjsg tmp |= bits;
456c349dbc7Sjsg WREG32(reg, tmp);
457c349dbc7Sjsg }
458c349dbc7Sjsg break;
459c349dbc7Sjsg default:
460c349dbc7Sjsg break;
461c349dbc7Sjsg }
462c349dbc7Sjsg
463c349dbc7Sjsg return 0;
464c349dbc7Sjsg }
465c349dbc7Sjsg
gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned int type,enum amdgpu_interrupt_state state)466fb4d8502Sjsg static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
467fb4d8502Sjsg struct amdgpu_irq_src *src,
468f005ef32Sjsg unsigned int type,
469fb4d8502Sjsg enum amdgpu_interrupt_state state)
470fb4d8502Sjsg {
471fb4d8502Sjsg struct amdgpu_vmhub *hub;
472fb4d8502Sjsg u32 tmp, reg, bits, i, j;
473fb4d8502Sjsg
474fb4d8502Sjsg bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
475fb4d8502Sjsg VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476fb4d8502Sjsg VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477fb4d8502Sjsg VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478fb4d8502Sjsg VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479fb4d8502Sjsg VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
480fb4d8502Sjsg VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
481fb4d8502Sjsg
482fb4d8502Sjsg switch (state) {
483fb4d8502Sjsg case AMDGPU_IRQ_STATE_DISABLE:
484f005ef32Sjsg for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
485fb4d8502Sjsg hub = &adev->vmhub[j];
486fb4d8502Sjsg for (i = 0; i < 16; i++) {
487fb4d8502Sjsg reg = hub->vm_context0_cntl + i;
4881bb76ff1Sjsg
489f005ef32Sjsg /* This works because this interrupt is only
490f005ef32Sjsg * enabled at init/resume and disabled in
491f005ef32Sjsg * fini/suspend, so the overall state doesn't
492f005ef32Sjsg * change over the course of suspend/resume.
493f005ef32Sjsg */
494f005ef32Sjsg if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
495f005ef32Sjsg continue;
496f005ef32Sjsg
497f005ef32Sjsg if (j >= AMDGPU_MMHUB0(0))
4981bb76ff1Sjsg tmp = RREG32_SOC15_IP(MMHUB, reg);
499f005ef32Sjsg else
500f005ef32Sjsg tmp = RREG32_SOC15_IP(GC, reg);
5011bb76ff1Sjsg
502fb4d8502Sjsg tmp &= ~bits;
5031bb76ff1Sjsg
504f005ef32Sjsg if (j >= AMDGPU_MMHUB0(0))
5051bb76ff1Sjsg WREG32_SOC15_IP(MMHUB, reg, tmp);
506f005ef32Sjsg else
507f005ef32Sjsg WREG32_SOC15_IP(GC, reg, tmp);
508fb4d8502Sjsg }
509fb4d8502Sjsg }
510fb4d8502Sjsg break;
511fb4d8502Sjsg case AMDGPU_IRQ_STATE_ENABLE:
512f005ef32Sjsg for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
513fb4d8502Sjsg hub = &adev->vmhub[j];
514fb4d8502Sjsg for (i = 0; i < 16; i++) {
515fb4d8502Sjsg reg = hub->vm_context0_cntl + i;
5161bb76ff1Sjsg
517f005ef32Sjsg /* This works because this interrupt is only
518f005ef32Sjsg * enabled at init/resume and disabled in
519f005ef32Sjsg * fini/suspend, so the overall state doesn't
520f005ef32Sjsg * change over the course of suspend/resume.
521f005ef32Sjsg */
522f005ef32Sjsg if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
523f005ef32Sjsg continue;
524f005ef32Sjsg
525f005ef32Sjsg if (j >= AMDGPU_MMHUB0(0))
5261bb76ff1Sjsg tmp = RREG32_SOC15_IP(MMHUB, reg);
527f005ef32Sjsg else
528f005ef32Sjsg tmp = RREG32_SOC15_IP(GC, reg);
5291bb76ff1Sjsg
530fb4d8502Sjsg tmp |= bits;
5311bb76ff1Sjsg
532f005ef32Sjsg if (j >= AMDGPU_MMHUB0(0))
5331bb76ff1Sjsg WREG32_SOC15_IP(MMHUB, reg, tmp);
534f005ef32Sjsg else
535f005ef32Sjsg WREG32_SOC15_IP(GC, reg, tmp);
536fb4d8502Sjsg }
537fb4d8502Sjsg }
5385ca02815Sjsg break;
539fb4d8502Sjsg default:
540fb4d8502Sjsg break;
541fb4d8502Sjsg }
542fb4d8502Sjsg
543fb4d8502Sjsg return 0;
544fb4d8502Sjsg }
545fb4d8502Sjsg
gmc_v9_0_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)546fb4d8502Sjsg static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
547fb4d8502Sjsg struct amdgpu_irq_src *source,
548fb4d8502Sjsg struct amdgpu_iv_entry *entry)
549fb4d8502Sjsg {
550c349dbc7Sjsg bool retry_fault = !!(entry->src_data[1] & 0x80);
5515ca02815Sjsg bool write_fault = !!(entry->src_data[1] & 0x20);
552ad8b1aafSjsg uint32_t status = 0, cid = 0, rw = 0;
5535ca02815Sjsg struct amdgpu_task_info task_info;
5545ca02815Sjsg struct amdgpu_vmhub *hub;
555ad8b1aafSjsg const char *mmhub_cid;
5565ca02815Sjsg const char *hub_name;
5575ca02815Sjsg u64 addr;
558f005ef32Sjsg uint32_t cam_index = 0;
559f005ef32Sjsg int ret, xcc_id = 0;
560f005ef32Sjsg uint32_t node_id;
561f005ef32Sjsg
562f005ef32Sjsg node_id = entry->node_id;
563fb4d8502Sjsg
564fb4d8502Sjsg addr = (u64)entry->src_data[0] << 12;
565fb4d8502Sjsg addr |= ((u64)entry->src_data[1] & 0xf) << 44;
566fb4d8502Sjsg
567f005ef32Sjsg if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
568f005ef32Sjsg hub_name = "mmhub0";
569f005ef32Sjsg hub = &adev->vmhub[AMDGPU_MMHUB0(node_id / 4)];
570f005ef32Sjsg } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
571f005ef32Sjsg hub_name = "mmhub1";
572f005ef32Sjsg hub = &adev->vmhub[AMDGPU_MMHUB1(0)];
573f005ef32Sjsg } else {
574f005ef32Sjsg hub_name = "gfxhub0";
575f005ef32Sjsg if (adev->gfx.funcs->ih_node_to_logical_xcc) {
576f005ef32Sjsg xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
577f005ef32Sjsg node_id);
578f005ef32Sjsg if (xcc_id < 0)
579f005ef32Sjsg xcc_id = 0;
580f005ef32Sjsg }
581f005ef32Sjsg hub = &adev->vmhub[xcc_id];
582f005ef32Sjsg }
5835ca02815Sjsg
584f005ef32Sjsg if (retry_fault) {
585f005ef32Sjsg if (adev->irq.retry_cam_enabled) {
586f005ef32Sjsg /* Delegate it to a different ring if the hardware hasn't
587f005ef32Sjsg * already done it.
588f005ef32Sjsg */
589f005ef32Sjsg if (entry->ih == &adev->irq.ih) {
590f005ef32Sjsg amdgpu_irq_delegate(adev, entry, 8);
591f005ef32Sjsg return 1;
592f005ef32Sjsg }
593f005ef32Sjsg
594f005ef32Sjsg cam_index = entry->src_data[2] & 0x3ff;
595f005ef32Sjsg
596f005ef32Sjsg ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
597f005ef32Sjsg addr, write_fault);
598f005ef32Sjsg WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
599f005ef32Sjsg if (ret)
600f005ef32Sjsg return 1;
601f005ef32Sjsg } else {
6025ca02815Sjsg /* Process it onyl if it's the first fault for this address */
6035ca02815Sjsg if (entry->ih != &adev->irq.ih_soft &&
6041bb76ff1Sjsg amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
605c349dbc7Sjsg entry->timestamp))
6065ca02815Sjsg return 1;
6075ca02815Sjsg
6085ca02815Sjsg /* Delegate it to a different ring if the hardware hasn't
6095ca02815Sjsg * already done it.
6105ca02815Sjsg */
6115ca02815Sjsg if (entry->ih == &adev->irq.ih) {
6125ca02815Sjsg amdgpu_irq_delegate(adev, entry, 8);
6135ca02815Sjsg return 1;
6145ca02815Sjsg }
6155ca02815Sjsg
6165ca02815Sjsg /* Try to handle the recoverable page faults by filling page
6175ca02815Sjsg * tables
6185ca02815Sjsg */
619f005ef32Sjsg if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
620f005ef32Sjsg addr, write_fault))
6215ca02815Sjsg return 1;
6225ca02815Sjsg }
623f005ef32Sjsg }
6245ca02815Sjsg
6255ca02815Sjsg if (!printk_ratelimit())
6265ca02815Sjsg return 0;
627c349dbc7Sjsg
628c349dbc7Sjsg
629c349dbc7Sjsg memset(&task_info, 0, sizeof(struct amdgpu_task_info));
630fb4d8502Sjsg amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
631fb4d8502Sjsg
632fb4d8502Sjsg dev_err(adev->dev,
633f005ef32Sjsg "[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n",
634c349dbc7Sjsg hub_name, retry_fault ? "retry" : "no-retry",
635fb4d8502Sjsg entry->src_id, entry->ring_id, entry->vmid,
636fb4d8502Sjsg entry->pasid, task_info.process_name, task_info.tgid,
637fb4d8502Sjsg task_info.task_name, task_info.pid);
6385ca02815Sjsg dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
6395ca02815Sjsg addr, entry->client_id,
6405ca02815Sjsg soc15_ih_clientid_name[entry->client_id]);
6415ca02815Sjsg
642f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
643f005ef32Sjsg dev_err(adev->dev, " cookie node_id %d fault from die %s%d%s\n",
644f005ef32Sjsg node_id, node_id % 4 == 3 ? "RSV" : "AID", node_id / 4,
645f005ef32Sjsg node_id % 4 == 1 ? ".XCD0" : node_id % 4 == 2 ? ".XCD1" : "");
646f005ef32Sjsg
6475ca02815Sjsg if (amdgpu_sriov_vf(adev))
6485ca02815Sjsg return 0;
6495ca02815Sjsg
6505ca02815Sjsg /*
6515ca02815Sjsg * Issue a dummy read to wait for the status register to
6525ca02815Sjsg * be updated to avoid reading an incorrect value due to
6535ca02815Sjsg * the new fast GRBM interface.
6545ca02815Sjsg */
655f005ef32Sjsg if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&
6561bb76ff1Sjsg (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2)))
6575ca02815Sjsg RREG32(hub->vm_l2_pro_fault_status);
6585ca02815Sjsg
6595ca02815Sjsg status = RREG32(hub->vm_l2_pro_fault_status);
6605ca02815Sjsg cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
6615ca02815Sjsg rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
6625ca02815Sjsg WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
6635ca02815Sjsg
664fb4d8502Sjsg dev_err(adev->dev,
665fb4d8502Sjsg "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
666fb4d8502Sjsg status);
667f005ef32Sjsg if (entry->vmid_src == AMDGPU_GFXHUB(0)) {
668ad8b1aafSjsg dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
6695ca02815Sjsg cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
6705ca02815Sjsg gfxhub_client_ids[cid],
671ad8b1aafSjsg cid);
672ad8b1aafSjsg } else {
6731bb76ff1Sjsg switch (adev->ip_versions[MMHUB_HWIP][0]) {
6741bb76ff1Sjsg case IP_VERSION(9, 0, 0):
675ad8b1aafSjsg mmhub_cid = mmhub_client_ids_vega10[cid][rw];
676ad8b1aafSjsg break;
6771bb76ff1Sjsg case IP_VERSION(9, 3, 0):
678ad8b1aafSjsg mmhub_cid = mmhub_client_ids_vega12[cid][rw];
679ad8b1aafSjsg break;
6801bb76ff1Sjsg case IP_VERSION(9, 4, 0):
681ad8b1aafSjsg mmhub_cid = mmhub_client_ids_vega20[cid][rw];
682ad8b1aafSjsg break;
6831bb76ff1Sjsg case IP_VERSION(9, 4, 1):
684ad8b1aafSjsg mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
685ad8b1aafSjsg break;
6861bb76ff1Sjsg case IP_VERSION(9, 1, 0):
6871bb76ff1Sjsg case IP_VERSION(9, 2, 0):
688ad8b1aafSjsg mmhub_cid = mmhub_client_ids_raven[cid][rw];
689ad8b1aafSjsg break;
6901bb76ff1Sjsg case IP_VERSION(1, 5, 0):
6911bb76ff1Sjsg case IP_VERSION(2, 4, 0):
692ad8b1aafSjsg mmhub_cid = mmhub_client_ids_renoir[cid][rw];
693ad8b1aafSjsg break;
694f005ef32Sjsg case IP_VERSION(1, 8, 0):
6951bb76ff1Sjsg case IP_VERSION(9, 4, 2):
6965ca02815Sjsg mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
6975ca02815Sjsg break;
698ad8b1aafSjsg default:
699ad8b1aafSjsg mmhub_cid = NULL;
700ad8b1aafSjsg break;
701ad8b1aafSjsg }
702ad8b1aafSjsg dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
703ad8b1aafSjsg mmhub_cid ? mmhub_cid : "unknown", cid);
704ad8b1aafSjsg }
705c349dbc7Sjsg dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
706c349dbc7Sjsg REG_GET_FIELD(status,
707c349dbc7Sjsg VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
708c349dbc7Sjsg dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
709c349dbc7Sjsg REG_GET_FIELD(status,
710c349dbc7Sjsg VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
711c349dbc7Sjsg dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
712c349dbc7Sjsg REG_GET_FIELD(status,
713c349dbc7Sjsg VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
714c349dbc7Sjsg dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
715c349dbc7Sjsg REG_GET_FIELD(status,
716c349dbc7Sjsg VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
717ad8b1aafSjsg dev_err(adev->dev, "\t RW: 0x%x\n", rw);
718fb4d8502Sjsg return 0;
719fb4d8502Sjsg }
720fb4d8502Sjsg
721fb4d8502Sjsg static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
722fb4d8502Sjsg .set = gmc_v9_0_vm_fault_interrupt_state,
723fb4d8502Sjsg .process = gmc_v9_0_process_interrupt,
724fb4d8502Sjsg };
725fb4d8502Sjsg
726c349dbc7Sjsg
727c349dbc7Sjsg static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
728c349dbc7Sjsg .set = gmc_v9_0_ecc_interrupt_state,
729c349dbc7Sjsg .process = amdgpu_umc_process_ecc_irq,
730c349dbc7Sjsg };
731c349dbc7Sjsg
gmc_v9_0_set_irq_funcs(struct amdgpu_device * adev)732fb4d8502Sjsg static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
733fb4d8502Sjsg {
734fb4d8502Sjsg adev->gmc.vm_fault.num_types = 1;
735fb4d8502Sjsg adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
736c349dbc7Sjsg
7375ca02815Sjsg if (!amdgpu_sriov_vf(adev) &&
7385ca02815Sjsg !adev->gmc.xgmi.connected_to_cpu) {
739c349dbc7Sjsg adev->gmc.ecc_irq.num_types = 1;
740c349dbc7Sjsg adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
741c349dbc7Sjsg }
742fb4d8502Sjsg }
743fb4d8502Sjsg
gmc_v9_0_get_invalidate_req(unsigned int vmid,uint32_t flush_type)744c349dbc7Sjsg static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
745c349dbc7Sjsg uint32_t flush_type)
746fb4d8502Sjsg {
747fb4d8502Sjsg u32 req = 0;
748fb4d8502Sjsg
749fb4d8502Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
750fb4d8502Sjsg PER_VMID_INVALIDATE_REQ, 1 << vmid);
751c349dbc7Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
752fb4d8502Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
753fb4d8502Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
754fb4d8502Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
755fb4d8502Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
756fb4d8502Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
757fb4d8502Sjsg req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
758fb4d8502Sjsg CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
759fb4d8502Sjsg
760fb4d8502Sjsg return req;
761fb4d8502Sjsg }
762fb4d8502Sjsg
763c349dbc7Sjsg /**
764c349dbc7Sjsg * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
765c349dbc7Sjsg *
766c349dbc7Sjsg * @adev: amdgpu_device pointer
767c349dbc7Sjsg * @vmhub: vmhub type
768c349dbc7Sjsg *
769c349dbc7Sjsg */
gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device * adev,uint32_t vmhub)770c349dbc7Sjsg static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
771c349dbc7Sjsg uint32_t vmhub)
772c349dbc7Sjsg {
773f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
774f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
7755ca02815Sjsg return false;
7765ca02815Sjsg
777f005ef32Sjsg return ((vmhub == AMDGPU_MMHUB0(0) ||
778f005ef32Sjsg vmhub == AMDGPU_MMHUB1(0)) &&
779c349dbc7Sjsg (!amdgpu_sriov_vf(adev)) &&
780ad8b1aafSjsg (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
781ad8b1aafSjsg (adev->apu_flags & AMD_APU_IS_PICASSO))));
782c349dbc7Sjsg }
783c349dbc7Sjsg
gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device * adev,uint8_t vmid,uint16_t * p_pasid)784c349dbc7Sjsg static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
785c349dbc7Sjsg uint8_t vmid, uint16_t *p_pasid)
786c349dbc7Sjsg {
787c349dbc7Sjsg uint32_t value;
788c349dbc7Sjsg
789c349dbc7Sjsg value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
790c349dbc7Sjsg + vmid);
791c349dbc7Sjsg *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
792c349dbc7Sjsg
793c349dbc7Sjsg return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
794c349dbc7Sjsg }
795c349dbc7Sjsg
796fb4d8502Sjsg /*
797fb4d8502Sjsg * GART
798fb4d8502Sjsg * VMID 0 is the physical GPU addresses as used by the kernel.
799fb4d8502Sjsg * VMIDs 1-15 are used for userspace clients and are handled
800fb4d8502Sjsg * by the amdgpu vm/hsa code.
801fb4d8502Sjsg */
802fb4d8502Sjsg
803fb4d8502Sjsg /**
804c349dbc7Sjsg * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
805fb4d8502Sjsg *
806fb4d8502Sjsg * @adev: amdgpu_device pointer
807fb4d8502Sjsg * @vmid: vm instance to flush
8085ca02815Sjsg * @vmhub: which hub to flush
809c349dbc7Sjsg * @flush_type: the flush type
810fb4d8502Sjsg *
811c349dbc7Sjsg * Flush the TLB for the requested page table using certain type.
812fb4d8502Sjsg */
gmc_v9_0_flush_gpu_tlb(struct amdgpu_device * adev,uint32_t vmid,uint32_t vmhub,uint32_t flush_type)813c349dbc7Sjsg static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
814c349dbc7Sjsg uint32_t vmhub, uint32_t flush_type)
815fb4d8502Sjsg {
816c349dbc7Sjsg bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
817f005ef32Sjsg const unsigned int eng = 17;
818c349dbc7Sjsg u32 j, inv_req, inv_req2, tmp;
819c349dbc7Sjsg struct amdgpu_vmhub *hub;
820c349dbc7Sjsg
821f005ef32Sjsg BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS);
822c349dbc7Sjsg
823c349dbc7Sjsg hub = &adev->vmhub[vmhub];
824c349dbc7Sjsg if (adev->gmc.xgmi.num_physical_nodes &&
8251bb76ff1Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0)) {
826c349dbc7Sjsg /* Vega20+XGMI caches PTEs in TC and TLB. Add a
827c349dbc7Sjsg * heavy-weight TLB flush (type 2), which flushes
828c349dbc7Sjsg * both. Due to a race condition with concurrent
829c349dbc7Sjsg * memory accesses using the same TLB cache line, we
830c349dbc7Sjsg * still need a second TLB flush after this.
831c349dbc7Sjsg */
832c349dbc7Sjsg inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
833c349dbc7Sjsg inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
834f005ef32Sjsg } else if (flush_type == 2 &&
835f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3) &&
836f005ef32Sjsg adev->rev_id == 0) {
837f005ef32Sjsg inv_req = gmc_v9_0_get_invalidate_req(vmid, 0);
838f005ef32Sjsg inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
839c349dbc7Sjsg } else {
840c349dbc7Sjsg inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
841c349dbc7Sjsg inv_req2 = 0;
842c349dbc7Sjsg }
843c349dbc7Sjsg
844c349dbc7Sjsg /* This is necessary for a HW workaround under SRIOV as well
845c349dbc7Sjsg * as GFXOFF under bare metal
846c349dbc7Sjsg */
847f005ef32Sjsg if (adev->gfx.kiq[0].ring.sched.ready &&
848c349dbc7Sjsg (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
8491bb76ff1Sjsg down_read_trylock(&adev->reset_domain->sem)) {
850ad8b1aafSjsg uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
851ad8b1aafSjsg uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
852c349dbc7Sjsg
853c349dbc7Sjsg amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
854c349dbc7Sjsg 1 << vmid);
8551bb76ff1Sjsg up_read(&adev->reset_domain->sem);
856c349dbc7Sjsg return;
857c349dbc7Sjsg }
858fb4d8502Sjsg
859fb4d8502Sjsg spin_lock(&adev->gmc.invalidate_lock);
860fb4d8502Sjsg
861c349dbc7Sjsg /*
862c349dbc7Sjsg * It may lose gpuvm invalidate acknowldege state across power-gating
863c349dbc7Sjsg * off cycle, add semaphore acquire before invalidation and semaphore
864c349dbc7Sjsg * release after invalidation to avoid entering power gated state
865c349dbc7Sjsg * to WA the Issue
866c349dbc7Sjsg */
867fb4d8502Sjsg
868c349dbc7Sjsg /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
869c349dbc7Sjsg if (use_semaphore) {
870fb4d8502Sjsg for (j = 0; j < adev->usec_timeout; j++) {
8711bb76ff1Sjsg /* a read return value of 1 means semaphore acquire */
872f005ef32Sjsg if (vmhub >= AMDGPU_MMHUB0(0))
8731bb76ff1Sjsg tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng);
874f005ef32Sjsg else
875f005ef32Sjsg tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng);
876c349dbc7Sjsg if (tmp & 0x1)
877fb4d8502Sjsg break;
878fb4d8502Sjsg udelay(1);
879fb4d8502Sjsg }
880c349dbc7Sjsg
881c349dbc7Sjsg if (j >= adev->usec_timeout)
882c349dbc7Sjsg DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
883c349dbc7Sjsg }
884c349dbc7Sjsg
885c349dbc7Sjsg do {
886f005ef32Sjsg if (vmhub >= AMDGPU_MMHUB0(0))
8871bb76ff1Sjsg WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req);
888f005ef32Sjsg else
889f005ef32Sjsg WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req);
890c349dbc7Sjsg
891c349dbc7Sjsg /*
892c349dbc7Sjsg * Issue a dummy read to wait for the ACK register to
893c349dbc7Sjsg * be cleared to avoid a false ACK due to the new fast
894c349dbc7Sjsg * GRBM interface.
895c349dbc7Sjsg */
896f005ef32Sjsg if ((vmhub == AMDGPU_GFXHUB(0)) &&
8971bb76ff1Sjsg (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2)))
898ad8b1aafSjsg RREG32_NO_KIQ(hub->vm_inv_eng0_req +
899ad8b1aafSjsg hub->eng_distance * eng);
900c349dbc7Sjsg
901c349dbc7Sjsg for (j = 0; j < adev->usec_timeout; j++) {
902f005ef32Sjsg if (vmhub >= AMDGPU_MMHUB0(0))
9031bb76ff1Sjsg tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_ack + hub->eng_distance * eng);
904f005ef32Sjsg else
905f005ef32Sjsg tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_ack + hub->eng_distance * eng);
906c349dbc7Sjsg if (tmp & (1 << vmid))
907c349dbc7Sjsg break;
908c349dbc7Sjsg udelay(1);
909c349dbc7Sjsg }
910c349dbc7Sjsg
911c349dbc7Sjsg inv_req = inv_req2;
912c349dbc7Sjsg inv_req2 = 0;
913c349dbc7Sjsg } while (inv_req);
914c349dbc7Sjsg
915c349dbc7Sjsg /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
9161bb76ff1Sjsg if (use_semaphore) {
917c349dbc7Sjsg /*
918c349dbc7Sjsg * add semaphore release after invalidation,
919c349dbc7Sjsg * write with 0 means semaphore release
920c349dbc7Sjsg */
921f005ef32Sjsg if (vmhub >= AMDGPU_MMHUB0(0))
9221bb76ff1Sjsg WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0);
923f005ef32Sjsg else
924f005ef32Sjsg WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0);
9251bb76ff1Sjsg }
926c349dbc7Sjsg
927c349dbc7Sjsg spin_unlock(&adev->gmc.invalidate_lock);
928c349dbc7Sjsg
929fb4d8502Sjsg if (j < adev->usec_timeout)
930c349dbc7Sjsg return;
931fb4d8502Sjsg
932fb4d8502Sjsg DRM_ERROR("Timeout waiting for VM flush ACK!\n");
933fb4d8502Sjsg }
934fb4d8502Sjsg
935c349dbc7Sjsg /**
936c349dbc7Sjsg * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
937c349dbc7Sjsg *
938c349dbc7Sjsg * @adev: amdgpu_device pointer
939c349dbc7Sjsg * @pasid: pasid to be flush
9405ca02815Sjsg * @flush_type: the flush type
9415ca02815Sjsg * @all_hub: flush all hubs
942f005ef32Sjsg * @inst: is used to select which instance of KIQ to use for the invalidation
943c349dbc7Sjsg *
944c349dbc7Sjsg * Flush the TLB for the requested pasid.
945c349dbc7Sjsg */
gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device * adev,uint16_t pasid,uint32_t flush_type,bool all_hub,uint32_t inst)946c349dbc7Sjsg static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
947c349dbc7Sjsg uint16_t pasid, uint32_t flush_type,
948f005ef32Sjsg bool all_hub, uint32_t inst)
949c349dbc7Sjsg {
950c349dbc7Sjsg int vmid, i;
951c349dbc7Sjsg signed long r;
952c349dbc7Sjsg uint32_t seq;
953c349dbc7Sjsg uint16_t queried_pasid;
954c349dbc7Sjsg bool ret;
955739bb310Sjsg u32 usec_timeout = amdgpu_sriov_vf(adev) ? SRIOV_USEC_TIMEOUT : adev->usec_timeout;
956f005ef32Sjsg struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
957f005ef32Sjsg struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
958c349dbc7Sjsg
959ad8b1aafSjsg if (amdgpu_in_reset(adev))
960c349dbc7Sjsg return -EIO;
961c349dbc7Sjsg
9621bb76ff1Sjsg if (ring->sched.ready && down_read_trylock(&adev->reset_domain->sem)) {
963c349dbc7Sjsg /* Vega20+XGMI caches PTEs in TC and TLB. Add a
964c349dbc7Sjsg * heavy-weight TLB flush (type 2), which flushes
965c349dbc7Sjsg * both. Due to a race condition with concurrent
966c349dbc7Sjsg * memory accesses using the same TLB cache line, we
967c349dbc7Sjsg * still need a second TLB flush after this.
968c349dbc7Sjsg */
969c349dbc7Sjsg bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
9701bb76ff1Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0));
971c349dbc7Sjsg /* 2 dwords flush + 8 dwords fence */
972c349dbc7Sjsg unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
973c349dbc7Sjsg
974c349dbc7Sjsg if (vega20_xgmi_wa)
975c349dbc7Sjsg ndw += kiq->pmf->invalidate_tlbs_size;
976c349dbc7Sjsg
977f005ef32Sjsg spin_lock(&adev->gfx.kiq[inst].ring_lock);
978c349dbc7Sjsg /* 2 dwords flush + 8 dwords fence */
979c349dbc7Sjsg amdgpu_ring_alloc(ring, ndw);
980c349dbc7Sjsg if (vega20_xgmi_wa)
981c349dbc7Sjsg kiq->pmf->kiq_invalidate_tlbs(ring,
982c349dbc7Sjsg pasid, 2, all_hub);
983f005ef32Sjsg
984f005ef32Sjsg if (flush_type == 2 &&
985f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3) &&
986f005ef32Sjsg adev->rev_id == 0)
987f005ef32Sjsg kiq->pmf->kiq_invalidate_tlbs(ring,
988f005ef32Sjsg pasid, 0, all_hub);
989f005ef32Sjsg
990c349dbc7Sjsg kiq->pmf->kiq_invalidate_tlbs(ring,
991c349dbc7Sjsg pasid, flush_type, all_hub);
992ad8b1aafSjsg r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
993ad8b1aafSjsg if (r) {
994ad8b1aafSjsg amdgpu_ring_undo(ring);
995f005ef32Sjsg spin_unlock(&adev->gfx.kiq[inst].ring_lock);
9961bb76ff1Sjsg up_read(&adev->reset_domain->sem);
997ad8b1aafSjsg return -ETIME;
998ad8b1aafSjsg }
999ad8b1aafSjsg
1000c349dbc7Sjsg amdgpu_ring_commit(ring);
1001f005ef32Sjsg spin_unlock(&adev->gfx.kiq[inst].ring_lock);
1002739bb310Sjsg r = amdgpu_fence_wait_polling(ring, seq, usec_timeout);
1003c349dbc7Sjsg if (r < 1) {
1004ad8b1aafSjsg dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
10051bb76ff1Sjsg up_read(&adev->reset_domain->sem);
1006c349dbc7Sjsg return -ETIME;
1007c349dbc7Sjsg }
10081bb76ff1Sjsg up_read(&adev->reset_domain->sem);
1009c349dbc7Sjsg return 0;
1010c349dbc7Sjsg }
1011c349dbc7Sjsg
1012c349dbc7Sjsg for (vmid = 1; vmid < 16; vmid++) {
1013c349dbc7Sjsg
1014c349dbc7Sjsg ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
1015c349dbc7Sjsg &queried_pasid);
1016c349dbc7Sjsg if (ret && queried_pasid == pasid) {
1017c349dbc7Sjsg if (all_hub) {
1018f005ef32Sjsg for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
1019c349dbc7Sjsg gmc_v9_0_flush_gpu_tlb(adev, vmid,
1020c349dbc7Sjsg i, flush_type);
1021c349dbc7Sjsg } else {
1022c349dbc7Sjsg gmc_v9_0_flush_gpu_tlb(adev, vmid,
1023f005ef32Sjsg AMDGPU_GFXHUB(0), flush_type);
1024c349dbc7Sjsg }
1025c349dbc7Sjsg break;
1026c349dbc7Sjsg }
1027c349dbc7Sjsg }
1028c349dbc7Sjsg
1029c349dbc7Sjsg return 0;
1030c349dbc7Sjsg
1031fb4d8502Sjsg }
1032fb4d8502Sjsg
gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring * ring,unsigned int vmid,uint64_t pd_addr)1033fb4d8502Sjsg static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
1034f005ef32Sjsg unsigned int vmid, uint64_t pd_addr)
1035fb4d8502Sjsg {
1036f005ef32Sjsg bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
1037fb4d8502Sjsg struct amdgpu_device *adev = ring->adev;
1038f005ef32Sjsg struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub];
1039c349dbc7Sjsg uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
1040f005ef32Sjsg unsigned int eng = ring->vm_inv_eng;
1041fb4d8502Sjsg
1042c349dbc7Sjsg /*
1043c349dbc7Sjsg * It may lose gpuvm invalidate acknowldege state across power-gating
1044c349dbc7Sjsg * off cycle, add semaphore acquire before invalidation and semaphore
1045c349dbc7Sjsg * release after invalidation to avoid entering power gated state
1046c349dbc7Sjsg * to WA the Issue
1047c349dbc7Sjsg */
1048c349dbc7Sjsg
1049c349dbc7Sjsg /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
1050c349dbc7Sjsg if (use_semaphore)
1051c349dbc7Sjsg /* a read return value of 1 means semaphore acuqire */
1052c349dbc7Sjsg amdgpu_ring_emit_reg_wait(ring,
1053ad8b1aafSjsg hub->vm_inv_eng0_sem +
1054ad8b1aafSjsg hub->eng_distance * eng, 0x1, 0x1);
1055fb4d8502Sjsg
1056ad8b1aafSjsg amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
1057ad8b1aafSjsg (hub->ctx_addr_distance * vmid),
1058fb4d8502Sjsg lower_32_bits(pd_addr));
1059fb4d8502Sjsg
1060ad8b1aafSjsg amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
1061ad8b1aafSjsg (hub->ctx_addr_distance * vmid),
1062fb4d8502Sjsg upper_32_bits(pd_addr));
1063fb4d8502Sjsg
1064ad8b1aafSjsg amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
1065ad8b1aafSjsg hub->eng_distance * eng,
1066ad8b1aafSjsg hub->vm_inv_eng0_ack +
1067ad8b1aafSjsg hub->eng_distance * eng,
1068fb4d8502Sjsg req, 1 << vmid);
1069fb4d8502Sjsg
1070c349dbc7Sjsg /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
1071c349dbc7Sjsg if (use_semaphore)
1072c349dbc7Sjsg /*
1073c349dbc7Sjsg * add semaphore release after invalidation,
1074c349dbc7Sjsg * write with 0 means semaphore release
1075c349dbc7Sjsg */
1076ad8b1aafSjsg amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
1077ad8b1aafSjsg hub->eng_distance * eng, 0);
1078c349dbc7Sjsg
1079fb4d8502Sjsg return pd_addr;
1080fb4d8502Sjsg }
1081fb4d8502Sjsg
gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring * ring,unsigned int vmid,unsigned int pasid)1082f005ef32Sjsg static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
1083f005ef32Sjsg unsigned int pasid)
1084fb4d8502Sjsg {
1085fb4d8502Sjsg struct amdgpu_device *adev = ring->adev;
1086fb4d8502Sjsg uint32_t reg;
1087fb4d8502Sjsg
1088c349dbc7Sjsg /* Do nothing because there's no lut register for mmhub1. */
1089f005ef32Sjsg if (ring->vm_hub == AMDGPU_MMHUB1(0))
1090c349dbc7Sjsg return;
1091c349dbc7Sjsg
1092f005ef32Sjsg if (ring->vm_hub == AMDGPU_GFXHUB(0))
1093fb4d8502Sjsg reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
1094fb4d8502Sjsg else
1095fb4d8502Sjsg reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
1096fb4d8502Sjsg
1097fb4d8502Sjsg amdgpu_ring_emit_wreg(ring, reg, pasid);
1098fb4d8502Sjsg }
1099fb4d8502Sjsg
1100fb4d8502Sjsg /*
1101fb4d8502Sjsg * PTE format on VEGA 10:
1102fb4d8502Sjsg * 63:59 reserved
1103fb4d8502Sjsg * 58:57 mtype
1104fb4d8502Sjsg * 56 F
1105fb4d8502Sjsg * 55 L
1106fb4d8502Sjsg * 54 P
1107fb4d8502Sjsg * 53 SW
1108fb4d8502Sjsg * 52 T
1109fb4d8502Sjsg * 50:48 reserved
1110fb4d8502Sjsg * 47:12 4k physical page base address
1111fb4d8502Sjsg * 11:7 fragment
1112fb4d8502Sjsg * 6 write
1113fb4d8502Sjsg * 5 read
1114fb4d8502Sjsg * 4 exe
1115fb4d8502Sjsg * 3 Z
1116fb4d8502Sjsg * 2 snooped
1117fb4d8502Sjsg * 1 system
1118fb4d8502Sjsg * 0 valid
1119fb4d8502Sjsg *
1120fb4d8502Sjsg * PDE format on VEGA 10:
1121fb4d8502Sjsg * 63:59 block fragment size
1122fb4d8502Sjsg * 58:55 reserved
1123fb4d8502Sjsg * 54 P
1124fb4d8502Sjsg * 53:48 reserved
1125fb4d8502Sjsg * 47:6 physical base address of PD or PTE
1126fb4d8502Sjsg * 5:3 reserved
1127fb4d8502Sjsg * 2 C
1128fb4d8502Sjsg * 1 system
1129fb4d8502Sjsg * 0 valid
1130fb4d8502Sjsg */
1131fb4d8502Sjsg
gmc_v9_0_map_mtype(struct amdgpu_device * adev,uint32_t flags)1132c349dbc7Sjsg static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1133fb4d8502Sjsg
1134fb4d8502Sjsg {
1135c349dbc7Sjsg switch (flags) {
1136fb4d8502Sjsg case AMDGPU_VM_MTYPE_DEFAULT:
1137c349dbc7Sjsg return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1138fb4d8502Sjsg case AMDGPU_VM_MTYPE_NC:
1139c349dbc7Sjsg return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1140fb4d8502Sjsg case AMDGPU_VM_MTYPE_WC:
1141c349dbc7Sjsg return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1142c349dbc7Sjsg case AMDGPU_VM_MTYPE_RW:
1143c349dbc7Sjsg return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1144fb4d8502Sjsg case AMDGPU_VM_MTYPE_CC:
1145c349dbc7Sjsg return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1146fb4d8502Sjsg case AMDGPU_VM_MTYPE_UC:
1147c349dbc7Sjsg return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1148fb4d8502Sjsg default:
1149c349dbc7Sjsg return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1150fb4d8502Sjsg }
1151fb4d8502Sjsg }
1152fb4d8502Sjsg
gmc_v9_0_get_vm_pde(struct amdgpu_device * adev,int level,uint64_t * addr,uint64_t * flags)1153fb4d8502Sjsg static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1154fb4d8502Sjsg uint64_t *addr, uint64_t *flags)
1155fb4d8502Sjsg {
1156c349dbc7Sjsg if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
11575ca02815Sjsg *addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
1158fb4d8502Sjsg BUG_ON(*addr & 0xFFFF00000000003FULL);
1159fb4d8502Sjsg
1160fb4d8502Sjsg if (!adev->gmc.translate_further)
1161fb4d8502Sjsg return;
1162fb4d8502Sjsg
1163fb4d8502Sjsg if (level == AMDGPU_VM_PDB1) {
1164fb4d8502Sjsg /* Set the block fragment size */
1165fb4d8502Sjsg if (!(*flags & AMDGPU_PDE_PTE))
1166fb4d8502Sjsg *flags |= AMDGPU_PDE_BFS(0x9);
1167fb4d8502Sjsg
1168fb4d8502Sjsg } else if (level == AMDGPU_VM_PDB0) {
11691bb76ff1Sjsg if (*flags & AMDGPU_PDE_PTE) {
1170fb4d8502Sjsg *flags &= ~AMDGPU_PDE_PTE;
11711bb76ff1Sjsg if (!(*flags & AMDGPU_PTE_VALID))
11721bb76ff1Sjsg *addr |= 1 << PAGE_SHIFT;
11731bb76ff1Sjsg } else {
1174fb4d8502Sjsg *flags |= AMDGPU_PTE_TF;
1175fb4d8502Sjsg }
1176fb4d8502Sjsg }
11771bb76ff1Sjsg }
1178fb4d8502Sjsg
gmc_v9_0_get_coherence_flags(struct amdgpu_device * adev,struct amdgpu_bo * bo,struct amdgpu_bo_va_mapping * mapping,uint64_t * flags)1179f005ef32Sjsg static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
1180f005ef32Sjsg struct amdgpu_bo *bo,
1181f005ef32Sjsg struct amdgpu_bo_va_mapping *mapping,
1182f005ef32Sjsg uint64_t *flags)
1183f005ef32Sjsg {
1184f005ef32Sjsg struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1185f005ef32Sjsg bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM;
1186f005ef32Sjsg bool coherent = bo->flags & AMDGPU_GEM_CREATE_COHERENT;
1187f005ef32Sjsg bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
1188f005ef32Sjsg struct amdgpu_vm *vm = mapping->bo_va->base.vm;
1189f005ef32Sjsg unsigned int mtype_local, mtype;
1190f005ef32Sjsg bool snoop = false;
1191f005ef32Sjsg bool is_local;
1192f005ef32Sjsg
1193f005ef32Sjsg switch (adev->ip_versions[GC_HWIP][0]) {
1194f005ef32Sjsg case IP_VERSION(9, 4, 1):
1195f005ef32Sjsg case IP_VERSION(9, 4, 2):
1196f005ef32Sjsg if (is_vram) {
1197f005ef32Sjsg if (bo_adev == adev) {
1198f005ef32Sjsg if (uncached)
1199f005ef32Sjsg mtype = MTYPE_UC;
1200f005ef32Sjsg else if (coherent)
1201f005ef32Sjsg mtype = MTYPE_CC;
1202f005ef32Sjsg else
1203f005ef32Sjsg mtype = MTYPE_RW;
1204f005ef32Sjsg /* FIXME: is this still needed? Or does
1205f005ef32Sjsg * amdgpu_ttm_tt_pde_flags already handle this?
1206f005ef32Sjsg */
1207f005ef32Sjsg if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
1208f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) &&
1209f005ef32Sjsg adev->gmc.xgmi.connected_to_cpu)
1210f005ef32Sjsg snoop = true;
1211f005ef32Sjsg } else {
1212f005ef32Sjsg if (uncached || coherent)
1213f005ef32Sjsg mtype = MTYPE_UC;
1214f005ef32Sjsg else
1215f005ef32Sjsg mtype = MTYPE_NC;
1216f005ef32Sjsg if (mapping->bo_va->is_xgmi)
1217f005ef32Sjsg snoop = true;
1218f005ef32Sjsg }
1219f005ef32Sjsg } else {
1220f005ef32Sjsg if (uncached || coherent)
1221f005ef32Sjsg mtype = MTYPE_UC;
1222f005ef32Sjsg else
1223f005ef32Sjsg mtype = MTYPE_NC;
1224f005ef32Sjsg /* FIXME: is this still needed? Or does
1225f005ef32Sjsg * amdgpu_ttm_tt_pde_flags already handle this?
1226f005ef32Sjsg */
1227f005ef32Sjsg snoop = true;
1228f005ef32Sjsg }
1229f005ef32Sjsg break;
1230f005ef32Sjsg case IP_VERSION(9, 4, 3):
1231f005ef32Sjsg /* Only local VRAM BOs or system memory on non-NUMA APUs
1232f005ef32Sjsg * can be assumed to be local in their entirety. Choose
1233f005ef32Sjsg * MTYPE_NC as safe fallback for all system memory BOs on
1234f005ef32Sjsg * NUMA systems. Their MTYPE can be overridden per-page in
1235f005ef32Sjsg * gmc_v9_0_override_vm_pte_flags.
1236f005ef32Sjsg */
1237f005ef32Sjsg mtype_local = MTYPE_RW;
1238f005ef32Sjsg if (amdgpu_mtype_local == 1) {
1239f005ef32Sjsg DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
1240f005ef32Sjsg mtype_local = MTYPE_NC;
1241f005ef32Sjsg } else if (amdgpu_mtype_local == 2) {
1242f005ef32Sjsg DRM_INFO_ONCE("Using MTYPE_CC for local memory\n");
1243f005ef32Sjsg mtype_local = MTYPE_CC;
1244f005ef32Sjsg } else {
1245f005ef32Sjsg DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
1246f005ef32Sjsg }
1247f005ef32Sjsg is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
1248f005ef32Sjsg num_possible_nodes() <= 1) ||
1249f005ef32Sjsg (is_vram && adev == bo_adev &&
1250f005ef32Sjsg KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id);
1251f005ef32Sjsg snoop = true;
1252f005ef32Sjsg if (uncached) {
1253f005ef32Sjsg mtype = MTYPE_UC;
1254f005ef32Sjsg } else if (adev->flags & AMD_IS_APU) {
1255f005ef32Sjsg mtype = is_local ? mtype_local : MTYPE_NC;
1256f005ef32Sjsg } else {
1257f005ef32Sjsg /* dGPU */
1258f005ef32Sjsg if (is_local)
1259f005ef32Sjsg mtype = mtype_local;
1260f005ef32Sjsg else if (is_vram)
1261f005ef32Sjsg mtype = MTYPE_NC;
1262f005ef32Sjsg else
1263f005ef32Sjsg mtype = MTYPE_UC;
1264f005ef32Sjsg }
1265f005ef32Sjsg
1266f005ef32Sjsg break;
1267f005ef32Sjsg default:
1268f005ef32Sjsg if (uncached || coherent)
1269f005ef32Sjsg mtype = MTYPE_UC;
1270f005ef32Sjsg else
1271f005ef32Sjsg mtype = MTYPE_NC;
1272f005ef32Sjsg
1273f005ef32Sjsg /* FIXME: is this still needed? Or does
1274f005ef32Sjsg * amdgpu_ttm_tt_pde_flags already handle this?
1275f005ef32Sjsg */
1276f005ef32Sjsg if (!is_vram)
1277f005ef32Sjsg snoop = true;
1278f005ef32Sjsg }
1279f005ef32Sjsg
1280f005ef32Sjsg if (mtype != MTYPE_NC)
1281f005ef32Sjsg *flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1282f005ef32Sjsg AMDGPU_PTE_MTYPE_VG10(mtype);
1283f005ef32Sjsg *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1284f005ef32Sjsg }
1285f005ef32Sjsg
gmc_v9_0_get_vm_pte(struct amdgpu_device * adev,struct amdgpu_bo_va_mapping * mapping,uint64_t * flags)1286c349dbc7Sjsg static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1287c349dbc7Sjsg struct amdgpu_bo_va_mapping *mapping,
1288c349dbc7Sjsg uint64_t *flags)
1289c349dbc7Sjsg {
1290f005ef32Sjsg struct amdgpu_bo *bo = mapping->bo_va->base.bo;
1291f005ef32Sjsg
1292c349dbc7Sjsg *flags &= ~AMDGPU_PTE_EXECUTABLE;
1293c349dbc7Sjsg *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1294c349dbc7Sjsg
1295c349dbc7Sjsg *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1296c349dbc7Sjsg *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1297c349dbc7Sjsg
1298c349dbc7Sjsg if (mapping->flags & AMDGPU_PTE_PRT) {
1299c349dbc7Sjsg *flags |= AMDGPU_PTE_PRT;
1300c349dbc7Sjsg *flags &= ~AMDGPU_PTE_VALID;
1301c349dbc7Sjsg }
1302c349dbc7Sjsg
1303f005ef32Sjsg if (bo && bo->tbo.resource)
1304f005ef32Sjsg gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.bo,
1305f005ef32Sjsg mapping, flags);
1306c349dbc7Sjsg }
1307c349dbc7Sjsg
gmc_v9_0_override_vm_pte_flags(struct amdgpu_device * adev,struct amdgpu_vm * vm,uint64_t addr,uint64_t * flags)1308f005ef32Sjsg static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
1309f005ef32Sjsg struct amdgpu_vm *vm,
1310f005ef32Sjsg uint64_t addr, uint64_t *flags)
1311f005ef32Sjsg {
1312f005ef32Sjsg int local_node, nid;
1313f005ef32Sjsg
1314f005ef32Sjsg /* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
1315f005ef32Sjsg * memory can use more efficient MTYPEs.
1316f005ef32Sjsg */
1317f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 3))
1318f005ef32Sjsg return;
1319f005ef32Sjsg
1320f005ef32Sjsg /* Only direct-mapped memory allows us to determine the NUMA node from
1321f005ef32Sjsg * the DMA address.
1322f005ef32Sjsg */
1323f005ef32Sjsg if (!adev->ram_is_direct_mapped) {
1324f005ef32Sjsg dev_dbg(adev->dev, "RAM is not direct mapped\n");
1325f005ef32Sjsg return;
1326f005ef32Sjsg }
1327f005ef32Sjsg
1328f005ef32Sjsg /* Only override mappings with MTYPE_NC, which is the safe default for
1329f005ef32Sjsg * cacheable memory.
1330f005ef32Sjsg */
1331f005ef32Sjsg if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1332f005ef32Sjsg AMDGPU_PTE_MTYPE_VG10(MTYPE_NC)) {
1333f005ef32Sjsg dev_dbg(adev->dev, "MTYPE is not NC\n");
1334f005ef32Sjsg return;
1335f005ef32Sjsg }
1336f005ef32Sjsg
1337f005ef32Sjsg /* FIXME: Only supported on native mode for now. For carve-out, the
1338f005ef32Sjsg * NUMA affinity of the GPU/VM needs to come from the PCI info because
1339f005ef32Sjsg * memory partitions are not associated with different NUMA nodes.
1340f005ef32Sjsg */
1341f005ef32Sjsg if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
1342f005ef32Sjsg local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
1343f005ef32Sjsg } else {
1344f005ef32Sjsg dev_dbg(adev->dev, "Only native mode APU is supported.\n");
1345f005ef32Sjsg return;
1346f005ef32Sjsg }
1347f005ef32Sjsg
1348f005ef32Sjsg /* Only handle real RAM. Mappings of PCIe resources don't have struct
1349f005ef32Sjsg * page or NUMA nodes.
1350f005ef32Sjsg */
1351f005ef32Sjsg #ifdef notyet
1352f005ef32Sjsg if (!page_is_ram(addr >> PAGE_SHIFT)) {
1353f005ef32Sjsg dev_dbg(adev->dev, "Page is not RAM.\n");
1354f005ef32Sjsg return;
1355f005ef32Sjsg }
1356f005ef32Sjsg #endif
1357f005ef32Sjsg nid = pfn_to_nid(addr >> PAGE_SHIFT);
1358f005ef32Sjsg dev_dbg(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n",
1359f005ef32Sjsg vm->mem_id, local_node, nid);
1360f005ef32Sjsg if (nid == local_node) {
1361f005ef32Sjsg uint64_t old_flags = *flags;
1362f005ef32Sjsg unsigned int mtype_local = MTYPE_RW;
1363f005ef32Sjsg
1364f005ef32Sjsg if (amdgpu_mtype_local == 1)
1365f005ef32Sjsg mtype_local = MTYPE_NC;
1366f005ef32Sjsg else if (amdgpu_mtype_local == 2)
1367f005ef32Sjsg mtype_local = MTYPE_CC;
1368f005ef32Sjsg
1369f005ef32Sjsg *flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1370f005ef32Sjsg AMDGPU_PTE_MTYPE_VG10(mtype_local);
1371f005ef32Sjsg dev_dbg(adev->dev, "flags updated from %llx to %llx\n",
1372f005ef32Sjsg old_flags, *flags);
1373f005ef32Sjsg }
1374f005ef32Sjsg }
1375f005ef32Sjsg
gmc_v9_0_get_vbios_fb_size(struct amdgpu_device * adev)1376f005ef32Sjsg static unsigned int gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1377ad8b1aafSjsg {
1378ad8b1aafSjsg u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1379f005ef32Sjsg unsigned int size;
1380ad8b1aafSjsg
1381c097090cSjsg /* TODO move to DC so GMC doesn't need to hard-code DCN registers */
1382c097090cSjsg
1383ad8b1aafSjsg if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1384ad8b1aafSjsg size = AMDGPU_VBIOS_VGA_ALLOCATION;
1385ad8b1aafSjsg } else {
1386ad8b1aafSjsg u32 viewport;
1387ad8b1aafSjsg
13881bb76ff1Sjsg switch (adev->ip_versions[DCE_HWIP][0]) {
13891bb76ff1Sjsg case IP_VERSION(1, 0, 0):
13901bb76ff1Sjsg case IP_VERSION(1, 0, 1):
1391ad8b1aafSjsg viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1392ad8b1aafSjsg size = (REG_GET_FIELD(viewport,
1393ad8b1aafSjsg HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1394ad8b1aafSjsg REG_GET_FIELD(viewport,
1395ad8b1aafSjsg HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1396ad8b1aafSjsg 4);
1397ad8b1aafSjsg break;
13981bb76ff1Sjsg case IP_VERSION(2, 1, 0):
1399c097090cSjsg viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
1400c097090cSjsg size = (REG_GET_FIELD(viewport,
1401c097090cSjsg HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1402c097090cSjsg REG_GET_FIELD(viewport,
1403c097090cSjsg HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1404c097090cSjsg 4);
1405c097090cSjsg break;
1406ad8b1aafSjsg default:
1407ad8b1aafSjsg viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1408ad8b1aafSjsg size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1409ad8b1aafSjsg REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1410ad8b1aafSjsg 4);
1411ad8b1aafSjsg break;
1412ad8b1aafSjsg }
1413ad8b1aafSjsg }
1414ad8b1aafSjsg
1415ad8b1aafSjsg return size;
1416ad8b1aafSjsg }
1417ad8b1aafSjsg
1418f005ef32Sjsg static enum amdgpu_memory_partition
gmc_v9_0_get_memory_partition(struct amdgpu_device * adev,u32 * supp_modes)1419f005ef32Sjsg gmc_v9_0_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes)
1420f005ef32Sjsg {
1421f005ef32Sjsg enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE;
1422f005ef32Sjsg
1423f005ef32Sjsg if (adev->nbio.funcs->get_memory_partition_mode)
1424f005ef32Sjsg mode = adev->nbio.funcs->get_memory_partition_mode(adev,
1425f005ef32Sjsg supp_modes);
1426f005ef32Sjsg
1427f005ef32Sjsg return mode;
1428f005ef32Sjsg }
1429f005ef32Sjsg
1430f005ef32Sjsg static enum amdgpu_memory_partition
gmc_v9_0_query_memory_partition(struct amdgpu_device * adev)1431f005ef32Sjsg gmc_v9_0_query_memory_partition(struct amdgpu_device *adev)
1432f005ef32Sjsg {
1433f005ef32Sjsg if (amdgpu_sriov_vf(adev))
1434f005ef32Sjsg return AMDGPU_NPS1_PARTITION_MODE;
1435f005ef32Sjsg
1436f005ef32Sjsg return gmc_v9_0_get_memory_partition(adev, NULL);
1437f005ef32Sjsg }
1438f005ef32Sjsg
1439fb4d8502Sjsg static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1440fb4d8502Sjsg .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1441c349dbc7Sjsg .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1442fb4d8502Sjsg .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1443fb4d8502Sjsg .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1444c349dbc7Sjsg .map_mtype = gmc_v9_0_map_mtype,
1445c349dbc7Sjsg .get_vm_pde = gmc_v9_0_get_vm_pde,
1446ad8b1aafSjsg .get_vm_pte = gmc_v9_0_get_vm_pte,
1447f005ef32Sjsg .override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
1448ad8b1aafSjsg .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1449f005ef32Sjsg .query_mem_partition_mode = &gmc_v9_0_query_memory_partition,
1450fb4d8502Sjsg };
1451fb4d8502Sjsg
gmc_v9_0_set_gmc_funcs(struct amdgpu_device * adev)1452fb4d8502Sjsg static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1453fb4d8502Sjsg {
1454fb4d8502Sjsg adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1455fb4d8502Sjsg }
1456fb4d8502Sjsg
gmc_v9_0_set_umc_funcs(struct amdgpu_device * adev)1457c349dbc7Sjsg static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1458c349dbc7Sjsg {
14591bb76ff1Sjsg switch (adev->ip_versions[UMC_HWIP][0]) {
14601bb76ff1Sjsg case IP_VERSION(6, 0, 0):
1461c349dbc7Sjsg adev->umc.funcs = &umc_v6_0_funcs;
1462c349dbc7Sjsg break;
14631bb76ff1Sjsg case IP_VERSION(6, 1, 1):
1464c349dbc7Sjsg adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1465c349dbc7Sjsg adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1466c349dbc7Sjsg adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1467c349dbc7Sjsg adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1468f005ef32Sjsg adev->umc.retire_unit = 1;
1469c349dbc7Sjsg adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
14701bb76ff1Sjsg adev->umc.ras = &umc_v6_1_ras;
1471c349dbc7Sjsg break;
14721bb76ff1Sjsg case IP_VERSION(6, 1, 2):
1473c349dbc7Sjsg adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1474c349dbc7Sjsg adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1475c349dbc7Sjsg adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1476c349dbc7Sjsg adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1477f005ef32Sjsg adev->umc.retire_unit = 1;
1478c349dbc7Sjsg adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
14791bb76ff1Sjsg adev->umc.ras = &umc_v6_1_ras;
14805ca02815Sjsg break;
14811bb76ff1Sjsg case IP_VERSION(6, 7, 0):
14821bb76ff1Sjsg adev->umc.max_ras_err_cnt_per_query =
14831bb76ff1Sjsg UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL;
14845ca02815Sjsg adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
14855ca02815Sjsg adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
14865ca02815Sjsg adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
1487f005ef32Sjsg adev->umc.retire_unit = (UMC_V6_7_NA_MAP_PA_NUM * 2);
14885ca02815Sjsg if (!adev->gmc.xgmi.connected_to_cpu)
14891bb76ff1Sjsg adev->umc.ras = &umc_v6_7_ras;
14905ca02815Sjsg if (1 & adev->smuio.funcs->get_die_id(adev))
14915ca02815Sjsg adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
14925ca02815Sjsg else
14935ca02815Sjsg adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1494c349dbc7Sjsg break;
1495c349dbc7Sjsg default:
1496c349dbc7Sjsg break;
1497c349dbc7Sjsg }
1498c349dbc7Sjsg }
1499c349dbc7Sjsg
gmc_v9_0_set_mmhub_funcs(struct amdgpu_device * adev)1500c349dbc7Sjsg static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1501c349dbc7Sjsg {
15021bb76ff1Sjsg switch (adev->ip_versions[MMHUB_HWIP][0]) {
15031bb76ff1Sjsg case IP_VERSION(9, 4, 1):
1504c349dbc7Sjsg adev->mmhub.funcs = &mmhub_v9_4_funcs;
1505c349dbc7Sjsg break;
15061bb76ff1Sjsg case IP_VERSION(9, 4, 2):
15075ca02815Sjsg adev->mmhub.funcs = &mmhub_v1_7_funcs;
15085ca02815Sjsg break;
1509f005ef32Sjsg case IP_VERSION(1, 8, 0):
1510f005ef32Sjsg adev->mmhub.funcs = &mmhub_v1_8_funcs;
1511f005ef32Sjsg break;
1512c349dbc7Sjsg default:
1513ad8b1aafSjsg adev->mmhub.funcs = &mmhub_v1_0_funcs;
1514ad8b1aafSjsg break;
1515ad8b1aafSjsg }
1516ad8b1aafSjsg }
1517ad8b1aafSjsg
gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device * adev)15185ca02815Sjsg static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1519ad8b1aafSjsg {
15201bb76ff1Sjsg switch (adev->ip_versions[MMHUB_HWIP][0]) {
15211bb76ff1Sjsg case IP_VERSION(9, 4, 0):
15221bb76ff1Sjsg adev->mmhub.ras = &mmhub_v1_0_ras;
15235ca02815Sjsg break;
15241bb76ff1Sjsg case IP_VERSION(9, 4, 1):
15251bb76ff1Sjsg adev->mmhub.ras = &mmhub_v9_4_ras;
15265ca02815Sjsg break;
15271bb76ff1Sjsg case IP_VERSION(9, 4, 2):
15281bb76ff1Sjsg adev->mmhub.ras = &mmhub_v1_7_ras;
1529ad8b1aafSjsg break;
1530f005ef32Sjsg case IP_VERSION(1, 8, 0):
1531f005ef32Sjsg adev->mmhub.ras = &mmhub_v1_8_ras;
1532f005ef32Sjsg break;
1533ad8b1aafSjsg default:
15345ca02815Sjsg /* mmhub ras is not available */
15355ca02815Sjsg break;
15365ca02815Sjsg }
15375ca02815Sjsg }
15385ca02815Sjsg
gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device * adev)15395ca02815Sjsg static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
15405ca02815Sjsg {
1541f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
1542f005ef32Sjsg adev->gfxhub.funcs = &gfxhub_v1_2_funcs;
1543f005ef32Sjsg else
1544ad8b1aafSjsg adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
15455ca02815Sjsg }
15465ca02815Sjsg
gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device * adev)15475ca02815Sjsg static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
15485ca02815Sjsg {
15491bb76ff1Sjsg adev->hdp.ras = &hdp_v4_0_ras;
15505ca02815Sjsg }
15515ca02815Sjsg
gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device * adev)1552f005ef32Sjsg static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev)
15535ca02815Sjsg {
1554f005ef32Sjsg struct amdgpu_mca *mca = &adev->mca;
1555f005ef32Sjsg
15561bb76ff1Sjsg /* is UMC the right IP to check for MCA? Maybe DF? */
15571bb76ff1Sjsg switch (adev->ip_versions[UMC_HWIP][0]) {
15581bb76ff1Sjsg case IP_VERSION(6, 7, 0):
1559f005ef32Sjsg if (!adev->gmc.xgmi.connected_to_cpu) {
1560f005ef32Sjsg mca->mp0.ras = &mca_v3_0_mp0_ras;
1561f005ef32Sjsg mca->mp1.ras = &mca_v3_0_mp1_ras;
1562f005ef32Sjsg mca->mpio.ras = &mca_v3_0_mpio_ras;
1563f005ef32Sjsg }
15645ca02815Sjsg break;
15655ca02815Sjsg default:
1566c349dbc7Sjsg break;
1567c349dbc7Sjsg }
1568c349dbc7Sjsg }
1569c349dbc7Sjsg
gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device * adev)1570f005ef32Sjsg static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
1571f005ef32Sjsg {
1572f005ef32Sjsg if (!adev->gmc.xgmi.connected_to_cpu)
1573f005ef32Sjsg adev->gmc.xgmi.ras = &xgmi_ras;
1574f005ef32Sjsg }
1575f005ef32Sjsg
gmc_v9_0_early_init(void * handle)1576fb4d8502Sjsg static int gmc_v9_0_early_init(void *handle)
1577fb4d8502Sjsg {
1578fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1579fb4d8502Sjsg
1580f005ef32Sjsg /*
1581f005ef32Sjsg * 9.4.0, 9.4.1 and 9.4.3 don't have XGMI defined
1582f005ef32Sjsg * in their IP discovery tables
1583f005ef32Sjsg */
1584f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0) ||
1585f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
1586f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
15875ca02815Sjsg adev->gmc.xgmi.supported = true;
15885ca02815Sjsg
15891bb76ff1Sjsg if (adev->ip_versions[XGMI_HWIP][0] == IP_VERSION(6, 1, 0)) {
15905ca02815Sjsg adev->gmc.xgmi.supported = true;
15915ca02815Sjsg adev->gmc.xgmi.connected_to_cpu =
15925ca02815Sjsg adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
15935ca02815Sjsg }
15945ca02815Sjsg
1595f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
1596f005ef32Sjsg enum amdgpu_pkg_type pkg_type =
1597f005ef32Sjsg adev->smuio.funcs->get_pkg_type(adev);
1598f005ef32Sjsg /* On GFXIP 9.4.3. APU, there is no physical VRAM domain present
1599f005ef32Sjsg * and the APU, can be in used two possible modes:
1600f005ef32Sjsg * - carveout mode
1601f005ef32Sjsg * - native APU mode
1602f005ef32Sjsg * "is_app_apu" can be used to identify the APU in the native
1603f005ef32Sjsg * mode.
1604f005ef32Sjsg */
1605f005ef32Sjsg #ifdef notyet
1606f005ef32Sjsg adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU &&
1607f005ef32Sjsg !pci_resource_len(adev->pdev, 0));
1608f005ef32Sjsg #else
1609f005ef32Sjsg adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU &&
1610f005ef32Sjsg !adev->fb_aper_size);
1611f005ef32Sjsg #endif
1612f005ef32Sjsg }
1613f005ef32Sjsg
1614fb4d8502Sjsg gmc_v9_0_set_gmc_funcs(adev);
1615fb4d8502Sjsg gmc_v9_0_set_irq_funcs(adev);
1616c349dbc7Sjsg gmc_v9_0_set_umc_funcs(adev);
1617c349dbc7Sjsg gmc_v9_0_set_mmhub_funcs(adev);
16185ca02815Sjsg gmc_v9_0_set_mmhub_ras_funcs(adev);
1619ad8b1aafSjsg gmc_v9_0_set_gfxhub_funcs(adev);
16205ca02815Sjsg gmc_v9_0_set_hdp_ras_funcs(adev);
1621f005ef32Sjsg gmc_v9_0_set_mca_ras_funcs(adev);
1622f005ef32Sjsg gmc_v9_0_set_xgmi_ras_funcs(adev);
1623fb4d8502Sjsg
1624fb4d8502Sjsg adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1625fb4d8502Sjsg adev->gmc.shared_aperture_end =
1626fb4d8502Sjsg adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1627fb4d8502Sjsg adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1628fb4d8502Sjsg adev->gmc.private_aperture_end =
1629fb4d8502Sjsg adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1630f005ef32Sjsg adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
16311bb76ff1Sjsg
1632fb4d8502Sjsg return 0;
1633fb4d8502Sjsg }
1634fb4d8502Sjsg
gmc_v9_0_late_init(void * handle)1635fb4d8502Sjsg static int gmc_v9_0_late_init(void *handle)
1636fb4d8502Sjsg {
1637fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1638fb4d8502Sjsg int r;
1639fb4d8502Sjsg
1640c349dbc7Sjsg r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1641c349dbc7Sjsg if (r)
1642fb4d8502Sjsg return r;
1643ad8b1aafSjsg
1644ad8b1aafSjsg /*
1645ad8b1aafSjsg * Workaround performance drop issue with VBIOS enables partial
1646ad8b1aafSjsg * writes, while disables HBM ECC for vega10.
1647ad8b1aafSjsg */
16481bb76ff1Sjsg if (!amdgpu_sriov_vf(adev) &&
16491bb76ff1Sjsg (adev->ip_versions[UMC_HWIP][0] == IP_VERSION(6, 0, 0))) {
16505ca02815Sjsg if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
16511bb76ff1Sjsg if (adev->df.funcs &&
16521bb76ff1Sjsg adev->df.funcs->enable_ecc_force_par_wr_rmw)
1653c349dbc7Sjsg adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1654ad8b1aafSjsg }
1655fb4d8502Sjsg }
1656c349dbc7Sjsg
16575ca02815Sjsg if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
16581bb76ff1Sjsg if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
16591bb76ff1Sjsg adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
16601bb76ff1Sjsg adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
16615ca02815Sjsg
16621bb76ff1Sjsg if (adev->hdp.ras && adev->hdp.ras->ras_block.hw_ops &&
16631bb76ff1Sjsg adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count)
16641bb76ff1Sjsg adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count(adev);
16655ca02815Sjsg }
1666c349dbc7Sjsg
1667c349dbc7Sjsg r = amdgpu_gmc_ras_late_init(adev);
1668c349dbc7Sjsg if (r)
1669c349dbc7Sjsg return r;
1670fb4d8502Sjsg
1671fb4d8502Sjsg return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1672fb4d8502Sjsg }
1673fb4d8502Sjsg
gmc_v9_0_vram_gtt_location(struct amdgpu_device * adev,struct amdgpu_gmc * mc)1674fb4d8502Sjsg static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1675fb4d8502Sjsg struct amdgpu_gmc *mc)
1676fb4d8502Sjsg {
16775ca02815Sjsg u64 base = adev->mmhub.funcs->get_fb_location(adev);
1678c349dbc7Sjsg
1679c349dbc7Sjsg /* add the xgmi offset of the physical node */
1680c349dbc7Sjsg base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
16815ca02815Sjsg if (adev->gmc.xgmi.connected_to_cpu) {
16825ca02815Sjsg amdgpu_gmc_sysvm_location(adev, mc);
16835ca02815Sjsg } else {
1684c349dbc7Sjsg amdgpu_gmc_vram_location(adev, mc, base);
1685c349dbc7Sjsg amdgpu_gmc_gart_location(adev, mc);
1686c349dbc7Sjsg amdgpu_gmc_agp_location(adev, mc);
16875ca02815Sjsg }
1688fb4d8502Sjsg /* base offset of vram pages */
1689ad8b1aafSjsg adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1690c349dbc7Sjsg
1691c349dbc7Sjsg /* XXX: add the xgmi offset of the physical node? */
1692c349dbc7Sjsg adev->vm_manager.vram_base_offset +=
1693c349dbc7Sjsg adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1694fb4d8502Sjsg }
1695fb4d8502Sjsg
1696fb4d8502Sjsg /**
1697fb4d8502Sjsg * gmc_v9_0_mc_init - initialize the memory controller driver params
1698fb4d8502Sjsg *
1699fb4d8502Sjsg * @adev: amdgpu_device pointer
1700fb4d8502Sjsg *
1701fb4d8502Sjsg * Look up the amount of vram, vram width, and decide how to place
1702fb4d8502Sjsg * vram and gart within the GPU's physical address space.
1703fb4d8502Sjsg * Returns 0 for success.
1704fb4d8502Sjsg */
gmc_v9_0_mc_init(struct amdgpu_device * adev)1705fb4d8502Sjsg static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1706fb4d8502Sjsg {
1707fb4d8502Sjsg int r;
1708fb4d8502Sjsg
1709fb4d8502Sjsg /* size in MB on si */
1710f005ef32Sjsg if (!adev->gmc.is_app_apu) {
1711fb4d8502Sjsg adev->gmc.mc_vram_size =
1712c349dbc7Sjsg adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1713f005ef32Sjsg } else {
1714f005ef32Sjsg DRM_DEBUG("Set mc_vram_size = 0 for APP APU\n");
1715f005ef32Sjsg adev->gmc.mc_vram_size = 0;
1716f005ef32Sjsg }
1717fb4d8502Sjsg adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1718fb4d8502Sjsg
17195ca02815Sjsg if (!(adev->flags & AMD_IS_APU) &&
17205ca02815Sjsg !adev->gmc.xgmi.connected_to_cpu) {
1721fb4d8502Sjsg r = amdgpu_device_resize_fb_bar(adev);
1722fb4d8502Sjsg if (r)
1723fb4d8502Sjsg return r;
1724fb4d8502Sjsg }
1725fb4d8502Sjsg adev->gmc.aper_base = adev->fb_aper_offset;
1726fb4d8502Sjsg adev->gmc.aper_size = adev->fb_aper_size;
1727fb4d8502Sjsg
172816fe02eaSjsg #ifdef CONFIG_X86_64
17295ca02815Sjsg /*
17305ca02815Sjsg * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi
17315ca02815Sjsg * interface can use VRAM through here as it appears system reserved
17325ca02815Sjsg * memory in host address space.
17335ca02815Sjsg *
17345ca02815Sjsg * For APUs, VRAM is just the stolen system memory and can be accessed
17355ca02815Sjsg * directly.
17365ca02815Sjsg *
17375ca02815Sjsg * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR.
17385ca02815Sjsg */
17395ca02815Sjsg
17405ca02815Sjsg /* check whether both host-gpu and gpu-gpu xgmi links exist */
1741f005ef32Sjsg if ((!amdgpu_sriov_vf(adev) &&
1742f005ef32Sjsg (adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
17435ca02815Sjsg (adev->gmc.xgmi.supported &&
17445ca02815Sjsg adev->gmc.xgmi.connected_to_cpu)) {
17455ca02815Sjsg adev->gmc.aper_base =
17465ca02815Sjsg adev->gfxhub.funcs->get_mc_fb_offset(adev) +
17475ca02815Sjsg adev->gmc.xgmi.physical_node_id *
17485ca02815Sjsg adev->gmc.xgmi.node_segment_size;
1749fb4d8502Sjsg adev->gmc.aper_size = adev->gmc.real_vram_size;
1750fb4d8502Sjsg }
17515ca02815Sjsg
1752fb4d8502Sjsg #endif
1753fb4d8502Sjsg adev->gmc.visible_vram_size = adev->gmc.aper_size;
1754fb4d8502Sjsg
1755fb4d8502Sjsg /* set the gart size */
1756fb4d8502Sjsg if (amdgpu_gart_size == -1) {
17571bb76ff1Sjsg switch (adev->ip_versions[GC_HWIP][0]) {
17581bb76ff1Sjsg case IP_VERSION(9, 0, 1): /* all engines support GPUVM */
17591bb76ff1Sjsg case IP_VERSION(9, 2, 1): /* all engines support GPUVM */
17601bb76ff1Sjsg case IP_VERSION(9, 4, 0):
17611bb76ff1Sjsg case IP_VERSION(9, 4, 1):
17621bb76ff1Sjsg case IP_VERSION(9, 4, 2):
1763f005ef32Sjsg case IP_VERSION(9, 4, 3):
1764fb4d8502Sjsg default:
1765fb4d8502Sjsg adev->gmc.gart_size = 512ULL << 20;
1766fb4d8502Sjsg break;
17671bb76ff1Sjsg case IP_VERSION(9, 1, 0): /* DCE SG support */
17681bb76ff1Sjsg case IP_VERSION(9, 2, 2): /* DCE SG support */
17691bb76ff1Sjsg case IP_VERSION(9, 3, 0):
1770fb4d8502Sjsg adev->gmc.gart_size = 1024ULL << 20;
1771fb4d8502Sjsg break;
1772fb4d8502Sjsg }
1773fb4d8502Sjsg } else {
1774fb4d8502Sjsg adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1775fb4d8502Sjsg }
1776fb4d8502Sjsg
17775ca02815Sjsg adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
17785ca02815Sjsg
1779fb4d8502Sjsg gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1780fb4d8502Sjsg
1781fb4d8502Sjsg return 0;
1782fb4d8502Sjsg }
1783fb4d8502Sjsg
gmc_v9_0_gart_init(struct amdgpu_device * adev)1784fb4d8502Sjsg static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1785fb4d8502Sjsg {
1786fb4d8502Sjsg int r;
1787fb4d8502Sjsg
1788c349dbc7Sjsg if (adev->gart.bo) {
1789fb4d8502Sjsg WARN(1, "VEGA10 PCIE GART already initialized\n");
1790fb4d8502Sjsg return 0;
1791fb4d8502Sjsg }
17925ca02815Sjsg
17935ca02815Sjsg if (adev->gmc.xgmi.connected_to_cpu) {
17945ca02815Sjsg adev->gmc.vmid0_page_table_depth = 1;
17955ca02815Sjsg adev->gmc.vmid0_page_table_block_size = 12;
17965ca02815Sjsg } else {
17975ca02815Sjsg adev->gmc.vmid0_page_table_depth = 0;
17985ca02815Sjsg adev->gmc.vmid0_page_table_block_size = 0;
17995ca02815Sjsg }
18005ca02815Sjsg
1801fb4d8502Sjsg /* Initialize common gart structure */
1802fb4d8502Sjsg r = amdgpu_gart_init(adev);
1803fb4d8502Sjsg if (r)
1804fb4d8502Sjsg return r;
1805fb4d8502Sjsg adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1806c349dbc7Sjsg adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1807fb4d8502Sjsg AMDGPU_PTE_EXECUTABLE;
18085ca02815Sjsg
1809f005ef32Sjsg if (!adev->gmc.real_vram_size) {
1810f005ef32Sjsg dev_info(adev->dev, "Put GART in system memory for APU\n");
1811f005ef32Sjsg r = amdgpu_gart_table_ram_alloc(adev);
1812f005ef32Sjsg if (r)
1813f005ef32Sjsg dev_err(adev->dev, "Failed to allocate GART in system memory\n");
1814f005ef32Sjsg } else {
18155ca02815Sjsg r = amdgpu_gart_table_vram_alloc(adev);
18165ca02815Sjsg if (r)
18175ca02815Sjsg return r;
18185ca02815Sjsg
1819f005ef32Sjsg if (adev->gmc.xgmi.connected_to_cpu)
18205ca02815Sjsg r = amdgpu_gmc_pdb0_alloc(adev);
18215ca02815Sjsg }
18225ca02815Sjsg
18235ca02815Sjsg return r;
1824fb4d8502Sjsg }
1825fb4d8502Sjsg
1826ad8b1aafSjsg /**
1827ad8b1aafSjsg * gmc_v9_0_save_registers - saves regs
1828ad8b1aafSjsg *
1829ad8b1aafSjsg * @adev: amdgpu_device pointer
1830ad8b1aafSjsg *
1831ad8b1aafSjsg * This saves potential register values that should be
1832ad8b1aafSjsg * restored upon resume
1833ad8b1aafSjsg */
gmc_v9_0_save_registers(struct amdgpu_device * adev)1834ad8b1aafSjsg static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1835fb4d8502Sjsg {
18361bb76ff1Sjsg if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) ||
18371bb76ff1Sjsg (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1)))
1838ad8b1aafSjsg adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1839fb4d8502Sjsg }
1840fb4d8502Sjsg
gmc_v9_0_validate_partition_info(struct amdgpu_device * adev)1841f005ef32Sjsg static bool gmc_v9_0_validate_partition_info(struct amdgpu_device *adev)
1842f005ef32Sjsg {
1843f005ef32Sjsg enum amdgpu_memory_partition mode;
1844f005ef32Sjsg u32 supp_modes;
1845f005ef32Sjsg bool valid;
1846f005ef32Sjsg
1847f005ef32Sjsg mode = gmc_v9_0_get_memory_partition(adev, &supp_modes);
1848f005ef32Sjsg
1849f005ef32Sjsg /* Mode detected by hardware not present in supported modes */
1850f005ef32Sjsg if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) &&
1851f005ef32Sjsg !(BIT(mode - 1) & supp_modes))
1852f005ef32Sjsg return false;
1853f005ef32Sjsg
1854f005ef32Sjsg switch (mode) {
1855f005ef32Sjsg case UNKNOWN_MEMORY_PARTITION_MODE:
1856f005ef32Sjsg case AMDGPU_NPS1_PARTITION_MODE:
1857f005ef32Sjsg valid = (adev->gmc.num_mem_partitions == 1);
1858f005ef32Sjsg break;
1859f005ef32Sjsg case AMDGPU_NPS2_PARTITION_MODE:
1860f005ef32Sjsg valid = (adev->gmc.num_mem_partitions == 2);
1861f005ef32Sjsg break;
1862f005ef32Sjsg case AMDGPU_NPS4_PARTITION_MODE:
1863f005ef32Sjsg valid = (adev->gmc.num_mem_partitions == 3 ||
1864f005ef32Sjsg adev->gmc.num_mem_partitions == 4);
1865f005ef32Sjsg break;
1866f005ef32Sjsg default:
1867f005ef32Sjsg valid = false;
1868f005ef32Sjsg }
1869f005ef32Sjsg
1870f005ef32Sjsg return valid;
1871f005ef32Sjsg }
1872f005ef32Sjsg
gmc_v9_0_is_node_present(int * node_ids,int num_ids,int nid)1873f005ef32Sjsg static bool gmc_v9_0_is_node_present(int *node_ids, int num_ids, int nid)
1874f005ef32Sjsg {
1875f005ef32Sjsg int i;
1876f005ef32Sjsg
1877f005ef32Sjsg /* Check if node with id 'nid' is present in 'node_ids' array */
1878f005ef32Sjsg for (i = 0; i < num_ids; ++i)
1879f005ef32Sjsg if (node_ids[i] == nid)
1880f005ef32Sjsg return true;
1881f005ef32Sjsg
1882f005ef32Sjsg return false;
1883f005ef32Sjsg }
1884f005ef32Sjsg
1885f005ef32Sjsg static void
gmc_v9_0_init_acpi_mem_ranges(struct amdgpu_device * adev,struct amdgpu_mem_partition_info * mem_ranges)1886f005ef32Sjsg gmc_v9_0_init_acpi_mem_ranges(struct amdgpu_device *adev,
1887f005ef32Sjsg struct amdgpu_mem_partition_info *mem_ranges)
1888f005ef32Sjsg {
1889f005ef32Sjsg int num_ranges = 0, ret, mem_groups;
1890f005ef32Sjsg struct amdgpu_numa_info numa_info;
1891f005ef32Sjsg int node_ids[MAX_MEM_RANGES];
1892f005ef32Sjsg int num_xcc, xcc_id;
1893f005ef32Sjsg uint32_t xcc_mask;
1894f005ef32Sjsg
1895f005ef32Sjsg num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1896f005ef32Sjsg xcc_mask = (1U << num_xcc) - 1;
1897f005ef32Sjsg mem_groups = hweight32(adev->aid_mask);
1898f005ef32Sjsg
1899f005ef32Sjsg for_each_inst(xcc_id, xcc_mask) {
1900f005ef32Sjsg ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info);
1901f005ef32Sjsg if (ret)
1902f005ef32Sjsg continue;
1903f005ef32Sjsg
1904f005ef32Sjsg if (numa_info.nid == NUMA_NO_NODE) {
1905f005ef32Sjsg mem_ranges[0].size = numa_info.size;
1906f005ef32Sjsg mem_ranges[0].numa.node = numa_info.nid;
1907f005ef32Sjsg num_ranges = 1;
1908f005ef32Sjsg break;
1909f005ef32Sjsg }
1910f005ef32Sjsg
1911f005ef32Sjsg if (gmc_v9_0_is_node_present(node_ids, num_ranges,
1912f005ef32Sjsg numa_info.nid))
1913f005ef32Sjsg continue;
1914f005ef32Sjsg
1915f005ef32Sjsg node_ids[num_ranges] = numa_info.nid;
1916f005ef32Sjsg mem_ranges[num_ranges].numa.node = numa_info.nid;
1917f005ef32Sjsg mem_ranges[num_ranges].size = numa_info.size;
1918f005ef32Sjsg ++num_ranges;
1919f005ef32Sjsg }
1920f005ef32Sjsg
1921f005ef32Sjsg adev->gmc.num_mem_partitions = num_ranges;
1922f005ef32Sjsg
1923f005ef32Sjsg /* If there is only partition, don't use entire size */
1924f005ef32Sjsg if (adev->gmc.num_mem_partitions == 1) {
1925f005ef32Sjsg mem_ranges[0].size = mem_ranges[0].size * (mem_groups - 1);
1926f005ef32Sjsg do_div(mem_ranges[0].size, mem_groups);
1927f005ef32Sjsg }
1928f005ef32Sjsg }
1929f005ef32Sjsg
1930f005ef32Sjsg static void
gmc_v9_0_init_sw_mem_ranges(struct amdgpu_device * adev,struct amdgpu_mem_partition_info * mem_ranges)1931f005ef32Sjsg gmc_v9_0_init_sw_mem_ranges(struct amdgpu_device *adev,
1932f005ef32Sjsg struct amdgpu_mem_partition_info *mem_ranges)
1933f005ef32Sjsg {
1934f005ef32Sjsg enum amdgpu_memory_partition mode;
1935f005ef32Sjsg u32 start_addr = 0, size;
1936f005ef32Sjsg int i;
1937f005ef32Sjsg
1938f005ef32Sjsg mode = gmc_v9_0_query_memory_partition(adev);
1939f005ef32Sjsg
1940f005ef32Sjsg switch (mode) {
1941f005ef32Sjsg case UNKNOWN_MEMORY_PARTITION_MODE:
1942f005ef32Sjsg case AMDGPU_NPS1_PARTITION_MODE:
1943f005ef32Sjsg adev->gmc.num_mem_partitions = 1;
1944f005ef32Sjsg break;
1945f005ef32Sjsg case AMDGPU_NPS2_PARTITION_MODE:
1946f005ef32Sjsg adev->gmc.num_mem_partitions = 2;
1947f005ef32Sjsg break;
1948f005ef32Sjsg case AMDGPU_NPS4_PARTITION_MODE:
1949f005ef32Sjsg if (adev->flags & AMD_IS_APU)
1950f005ef32Sjsg adev->gmc.num_mem_partitions = 3;
1951f005ef32Sjsg else
1952f005ef32Sjsg adev->gmc.num_mem_partitions = 4;
1953f005ef32Sjsg break;
1954f005ef32Sjsg default:
1955f005ef32Sjsg adev->gmc.num_mem_partitions = 1;
1956f005ef32Sjsg break;
1957f005ef32Sjsg }
1958f005ef32Sjsg
1959f005ef32Sjsg size = adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT;
1960f005ef32Sjsg size /= adev->gmc.num_mem_partitions;
1961f005ef32Sjsg
1962f005ef32Sjsg for (i = 0; i < adev->gmc.num_mem_partitions; ++i) {
1963f005ef32Sjsg mem_ranges[i].range.fpfn = start_addr;
1964f005ef32Sjsg mem_ranges[i].size = ((u64)size << AMDGPU_GPU_PAGE_SHIFT);
1965f005ef32Sjsg mem_ranges[i].range.lpfn = start_addr + size - 1;
1966f005ef32Sjsg start_addr += size;
1967f005ef32Sjsg }
1968f005ef32Sjsg
1969f005ef32Sjsg /* Adjust the last one */
1970f005ef32Sjsg mem_ranges[adev->gmc.num_mem_partitions - 1].range.lpfn =
1971f005ef32Sjsg (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1;
1972f005ef32Sjsg mem_ranges[adev->gmc.num_mem_partitions - 1].size =
1973f005ef32Sjsg adev->gmc.real_vram_size -
1974f005ef32Sjsg ((u64)mem_ranges[adev->gmc.num_mem_partitions - 1].range.fpfn
1975f005ef32Sjsg << AMDGPU_GPU_PAGE_SHIFT);
1976f005ef32Sjsg }
1977f005ef32Sjsg
gmc_v9_0_init_mem_ranges(struct amdgpu_device * adev)1978f005ef32Sjsg static int gmc_v9_0_init_mem_ranges(struct amdgpu_device *adev)
1979f005ef32Sjsg {
1980f005ef32Sjsg bool valid;
1981f005ef32Sjsg
1982f005ef32Sjsg adev->gmc.mem_partitions = kzalloc(
1983f005ef32Sjsg MAX_MEM_RANGES * sizeof(struct amdgpu_mem_partition_info),
1984f005ef32Sjsg GFP_KERNEL);
1985f005ef32Sjsg
1986f005ef32Sjsg if (!adev->gmc.mem_partitions)
1987f005ef32Sjsg return -ENOMEM;
1988f005ef32Sjsg
1989f005ef32Sjsg /* TODO : Get the range from PSP/Discovery for dGPU */
1990f005ef32Sjsg if (adev->gmc.is_app_apu)
1991f005ef32Sjsg gmc_v9_0_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions);
1992f005ef32Sjsg else
1993f005ef32Sjsg gmc_v9_0_init_sw_mem_ranges(adev, adev->gmc.mem_partitions);
1994f005ef32Sjsg
1995f005ef32Sjsg if (amdgpu_sriov_vf(adev))
1996f005ef32Sjsg valid = true;
1997f005ef32Sjsg else
1998f005ef32Sjsg valid = gmc_v9_0_validate_partition_info(adev);
1999f005ef32Sjsg if (!valid) {
2000f005ef32Sjsg /* TODO: handle invalid case */
2001f005ef32Sjsg dev_WARN(adev->dev,
2002f005ef32Sjsg "Mem ranges not matching with hardware config");
2003f005ef32Sjsg }
2004f005ef32Sjsg
2005f005ef32Sjsg return 0;
2006f005ef32Sjsg }
2007f005ef32Sjsg
gmc_v9_4_3_init_vram_info(struct amdgpu_device * adev)2008f005ef32Sjsg static void gmc_v9_4_3_init_vram_info(struct amdgpu_device *adev)
2009f005ef32Sjsg {
2010f005ef32Sjsg static const u32 regBIF_BIOS_SCRATCH_4 = 0x50;
2011f005ef32Sjsg u32 vram_info;
2012f005ef32Sjsg
2013f005ef32Sjsg if (!amdgpu_sriov_vf(adev)) {
2014f005ef32Sjsg vram_info = RREG32(regBIF_BIOS_SCRATCH_4);
2015f005ef32Sjsg adev->gmc.vram_vendor = vram_info & 0xF;
2016f005ef32Sjsg }
2017f005ef32Sjsg adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
2018f005ef32Sjsg adev->gmc.vram_width = 128 * 64;
2019f005ef32Sjsg }
2020f005ef32Sjsg
gmc_v9_0_sw_init(void * handle)2021fb4d8502Sjsg static int gmc_v9_0_sw_init(void *handle)
2022fb4d8502Sjsg {
20231bb76ff1Sjsg int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
2024fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2025f005ef32Sjsg unsigned long inst_mask = adev->aid_mask;
2026fb4d8502Sjsg
2027ad8b1aafSjsg adev->gfxhub.funcs->init(adev);
2028ad8b1aafSjsg
2029ad8b1aafSjsg adev->mmhub.funcs->init(adev);
2030fb4d8502Sjsg
203163b35fb2Sjsg mtx_init(&adev->gmc.invalidate_lock, IPL_NONE);
2032fb4d8502Sjsg
2033f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
2034f005ef32Sjsg gmc_v9_4_3_init_vram_info(adev);
2035f005ef32Sjsg } else if (!adev->bios) {
2036f005ef32Sjsg if (adev->flags & AMD_IS_APU) {
2037f005ef32Sjsg adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;
2038f005ef32Sjsg adev->gmc.vram_width = 64 * 64;
2039f005ef32Sjsg } else {
2040f005ef32Sjsg adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
2041f005ef32Sjsg adev->gmc.vram_width = 128 * 64;
2042f005ef32Sjsg }
2043f005ef32Sjsg } else {
2044c349dbc7Sjsg r = amdgpu_atomfirmware_get_vram_info(adev,
2045c349dbc7Sjsg &vram_width, &vram_type, &vram_vendor);
2046c349dbc7Sjsg if (amdgpu_sriov_vf(adev))
2047c349dbc7Sjsg /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
2048c349dbc7Sjsg * and DF related registers is not readable, seems hardcord is the
2049c349dbc7Sjsg * only way to set the correct vram_width
2050c349dbc7Sjsg */
2051c349dbc7Sjsg adev->gmc.vram_width = 2048;
2052c349dbc7Sjsg else if (amdgpu_emu_mode != 1)
2053c349dbc7Sjsg adev->gmc.vram_width = vram_width;
2054c349dbc7Sjsg
2055c349dbc7Sjsg if (!adev->gmc.vram_width) {
2056c349dbc7Sjsg int chansize, numchan;
2057c349dbc7Sjsg
2058c349dbc7Sjsg /* hbm memory channel size */
2059c349dbc7Sjsg if (adev->flags & AMD_IS_APU)
2060c349dbc7Sjsg chansize = 64;
2061c349dbc7Sjsg else
2062c349dbc7Sjsg chansize = 128;
20631bb76ff1Sjsg if (adev->df.funcs &&
20641bb76ff1Sjsg adev->df.funcs->get_hbm_channel_number) {
2065c349dbc7Sjsg numchan = adev->df.funcs->get_hbm_channel_number(adev);
2066c349dbc7Sjsg adev->gmc.vram_width = numchan * chansize;
2067c349dbc7Sjsg }
20681bb76ff1Sjsg }
2069c349dbc7Sjsg
2070c349dbc7Sjsg adev->gmc.vram_type = vram_type;
2071c349dbc7Sjsg adev->gmc.vram_vendor = vram_vendor;
2072f005ef32Sjsg }
20731bb76ff1Sjsg switch (adev->ip_versions[GC_HWIP][0]) {
20741bb76ff1Sjsg case IP_VERSION(9, 1, 0):
20751bb76ff1Sjsg case IP_VERSION(9, 2, 2):
2076f005ef32Sjsg set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2077f005ef32Sjsg set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2078c349dbc7Sjsg
2079fb4d8502Sjsg if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
2080fb4d8502Sjsg amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2081fb4d8502Sjsg } else {
2082fb4d8502Sjsg /* vm_size is 128TB + 512GB for legacy 3-level page support */
2083fb4d8502Sjsg amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
2084fb4d8502Sjsg adev->gmc.translate_further =
2085fb4d8502Sjsg adev->vm_manager.num_level > 1;
2086fb4d8502Sjsg }
2087fb4d8502Sjsg break;
20881bb76ff1Sjsg case IP_VERSION(9, 0, 1):
20891bb76ff1Sjsg case IP_VERSION(9, 2, 1):
20901bb76ff1Sjsg case IP_VERSION(9, 4, 0):
20911bb76ff1Sjsg case IP_VERSION(9, 3, 0):
20921bb76ff1Sjsg case IP_VERSION(9, 4, 2):
2093f005ef32Sjsg set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2094f005ef32Sjsg set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2095c349dbc7Sjsg
2096fb4d8502Sjsg /*
2097fb4d8502Sjsg * To fulfill 4-level page support,
2098fb4d8502Sjsg * vm size is 256TB (48bit), maximum size of Vega10,
2099fb4d8502Sjsg * block size 512 (9bit)
2100fb4d8502Sjsg */
2101c349dbc7Sjsg /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
2102c349dbc7Sjsg if (amdgpu_sriov_vf(adev))
2103c349dbc7Sjsg amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
2104c349dbc7Sjsg else
2105c349dbc7Sjsg amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
21061bb76ff1Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
21071bb76ff1Sjsg adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2108c349dbc7Sjsg break;
21091bb76ff1Sjsg case IP_VERSION(9, 4, 1):
2110f005ef32Sjsg set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2111f005ef32Sjsg set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2112f005ef32Sjsg set_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask);
2113c349dbc7Sjsg
2114c349dbc7Sjsg /* Keep the vm size same with Vega20 */
2115fb4d8502Sjsg amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
21161bb76ff1Sjsg adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2117fb4d8502Sjsg break;
2118f005ef32Sjsg case IP_VERSION(9, 4, 3):
2119f005ef32Sjsg bitmap_set(adev->vmhubs_mask, AMDGPU_GFXHUB(0),
2120f005ef32Sjsg NUM_XCC(adev->gfx.xcc_mask));
2121f005ef32Sjsg
2122f005ef32Sjsg inst_mask <<= AMDGPU_MMHUB0(0);
2123f005ef32Sjsg bitmap_or(adev->vmhubs_mask, adev->vmhubs_mask, &inst_mask, 32);
2124f005ef32Sjsg
2125f005ef32Sjsg amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2126f005ef32Sjsg adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2127f005ef32Sjsg break;
2128fb4d8502Sjsg default:
2129fb4d8502Sjsg break;
2130fb4d8502Sjsg }
2131fb4d8502Sjsg
2132fb4d8502Sjsg /* This interrupt is VMC page fault.*/
2133fb4d8502Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
2134fb4d8502Sjsg &adev->gmc.vm_fault);
2135c349dbc7Sjsg if (r)
2136c349dbc7Sjsg return r;
2137c349dbc7Sjsg
21381bb76ff1Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) {
2139c349dbc7Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
2140c349dbc7Sjsg &adev->gmc.vm_fault);
2141c349dbc7Sjsg if (r)
2142c349dbc7Sjsg return r;
2143c349dbc7Sjsg }
2144c349dbc7Sjsg
2145fb4d8502Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
2146fb4d8502Sjsg &adev->gmc.vm_fault);
2147fb4d8502Sjsg
2148fb4d8502Sjsg if (r)
2149fb4d8502Sjsg return r;
2150fb4d8502Sjsg
21515ca02815Sjsg if (!amdgpu_sriov_vf(adev) &&
21525ca02815Sjsg !adev->gmc.xgmi.connected_to_cpu) {
2153c349dbc7Sjsg /* interrupt sent to DF. */
2154c349dbc7Sjsg r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
2155c349dbc7Sjsg &adev->gmc.ecc_irq);
2156c349dbc7Sjsg if (r)
2157c349dbc7Sjsg return r;
2158c349dbc7Sjsg }
2159c349dbc7Sjsg
2160fb4d8502Sjsg /* Set the internal MC address mask
2161fb4d8502Sjsg * This is the max address of the GPU's
2162fb4d8502Sjsg * internal address space.
2163fb4d8502Sjsg */
2164fb4d8502Sjsg adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
2165fb4d8502Sjsg
2166f005ef32Sjsg dma_addr_bits = adev->ip_versions[GC_HWIP][0] >= IP_VERSION(9, 4, 2) ? 48:44;
21671bb76ff1Sjsg r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(dma_addr_bits));
2168fb4d8502Sjsg if (r) {
2169f005ef32Sjsg dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
2170c349dbc7Sjsg return r;
2171fb4d8502Sjsg }
21721bb76ff1Sjsg adev->need_swiotlb = drm_need_swiotlb(dma_addr_bits);
2173fb4d8502Sjsg
2174fb4d8502Sjsg r = gmc_v9_0_mc_init(adev);
2175fb4d8502Sjsg if (r)
2176fb4d8502Sjsg return r;
2177fb4d8502Sjsg
2178ad8b1aafSjsg amdgpu_gmc_get_vbios_allocations(adev);
2179fb4d8502Sjsg
2180f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
2181f005ef32Sjsg r = gmc_v9_0_init_mem_ranges(adev);
2182f005ef32Sjsg if (r)
2183f005ef32Sjsg return r;
2184f005ef32Sjsg }
2185f005ef32Sjsg
2186fb4d8502Sjsg /* Memory manager */
2187fb4d8502Sjsg r = amdgpu_bo_init(adev);
2188fb4d8502Sjsg if (r)
2189fb4d8502Sjsg return r;
2190fb4d8502Sjsg
2191fb4d8502Sjsg r = gmc_v9_0_gart_init(adev);
2192fb4d8502Sjsg if (r)
2193fb4d8502Sjsg return r;
2194fb4d8502Sjsg
2195fb4d8502Sjsg /*
2196fb4d8502Sjsg * number of VMs
2197fb4d8502Sjsg * VMID 0 is reserved for System
2198ad8b1aafSjsg * amdgpu graphics/compute will use VMIDs 1..n-1
2199ad8b1aafSjsg * amdkfd will use VMIDs n..15
2200ad8b1aafSjsg *
2201ad8b1aafSjsg * The first KFD VMID is 8 for GPUs with graphics, 3 for
2202ad8b1aafSjsg * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
2203ad8b1aafSjsg * for video processing.
2204fb4d8502Sjsg */
2205ad8b1aafSjsg adev->vm_manager.first_kfd_vmid =
22061bb76ff1Sjsg (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
2207f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
2208f005ef32Sjsg adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) ? 3 : 8;
2209fb4d8502Sjsg
2210fb4d8502Sjsg amdgpu_vm_manager_init(adev);
2211fb4d8502Sjsg
2212ad8b1aafSjsg gmc_v9_0_save_registers(adev);
2213ad8b1aafSjsg
2214f005ef32Sjsg r = amdgpu_gmc_ras_sw_init(adev);
2215f005ef32Sjsg if (r)
2216f005ef32Sjsg return r;
2217f005ef32Sjsg
2218f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
2219f005ef32Sjsg amdgpu_gmc_sysfs_init(adev);
2220f005ef32Sjsg
2221fb4d8502Sjsg return 0;
2222fb4d8502Sjsg }
2223fb4d8502Sjsg
gmc_v9_0_sw_fini(void * handle)2224fb4d8502Sjsg static int gmc_v9_0_sw_fini(void *handle)
2225fb4d8502Sjsg {
2226fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2227fb4d8502Sjsg
2228f005ef32Sjsg if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
2229f005ef32Sjsg amdgpu_gmc_sysfs_fini(adev);
2230f005ef32Sjsg
2231c349dbc7Sjsg amdgpu_gmc_ras_fini(adev);
2232fb4d8502Sjsg amdgpu_gem_force_release(adev);
2233fb4d8502Sjsg amdgpu_vm_manager_fini(adev);
2234f005ef32Sjsg if (!adev->gmc.real_vram_size) {
2235f005ef32Sjsg dev_info(adev->dev, "Put GART in system memory for APU free\n");
2236f005ef32Sjsg amdgpu_gart_table_ram_free(adev);
2237f005ef32Sjsg } else {
2238fb4d8502Sjsg amdgpu_gart_table_vram_free(adev);
2239f005ef32Sjsg }
2240ceb0ae7aSjsg amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
2241fb4d8502Sjsg amdgpu_bo_fini(adev);
2242fb4d8502Sjsg
2243f005ef32Sjsg adev->gmc.num_mem_partitions = 0;
2244f005ef32Sjsg kfree(adev->gmc.mem_partitions);
2245f005ef32Sjsg
2246fb4d8502Sjsg return 0;
2247fb4d8502Sjsg }
2248fb4d8502Sjsg
gmc_v9_0_init_golden_registers(struct amdgpu_device * adev)2249fb4d8502Sjsg static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
2250fb4d8502Sjsg {
2251fb4d8502Sjsg
22521bb76ff1Sjsg switch (adev->ip_versions[MMHUB_HWIP][0]) {
22531bb76ff1Sjsg case IP_VERSION(9, 0, 0):
2254c349dbc7Sjsg if (amdgpu_sriov_vf(adev))
2255c349dbc7Sjsg break;
2256ad8b1aafSjsg fallthrough;
22571bb76ff1Sjsg case IP_VERSION(9, 4, 0):
2258fb4d8502Sjsg soc15_program_register_sequence(adev,
2259fb4d8502Sjsg golden_settings_mmhub_1_0_0,
2260fb4d8502Sjsg ARRAY_SIZE(golden_settings_mmhub_1_0_0));
2261fb4d8502Sjsg soc15_program_register_sequence(adev,
2262fb4d8502Sjsg golden_settings_athub_1_0_0,
2263fb4d8502Sjsg ARRAY_SIZE(golden_settings_athub_1_0_0));
2264fb4d8502Sjsg break;
22651bb76ff1Sjsg case IP_VERSION(9, 1, 0):
22661bb76ff1Sjsg case IP_VERSION(9, 2, 0):
2267c349dbc7Sjsg /* TODO for renoir */
2268fb4d8502Sjsg soc15_program_register_sequence(adev,
2269fb4d8502Sjsg golden_settings_athub_1_0_0,
2270fb4d8502Sjsg ARRAY_SIZE(golden_settings_athub_1_0_0));
2271fb4d8502Sjsg break;
2272fb4d8502Sjsg default:
2273fb4d8502Sjsg break;
2274fb4d8502Sjsg }
2275fb4d8502Sjsg }
2276fb4d8502Sjsg
2277fb4d8502Sjsg /**
227822873acaSjsg * gmc_v9_0_restore_registers - restores regs
227922873acaSjsg *
228022873acaSjsg * @adev: amdgpu_device pointer
228122873acaSjsg *
228222873acaSjsg * This restores register values, saved at suspend.
228322873acaSjsg */
gmc_v9_0_restore_registers(struct amdgpu_device * adev)2284ad8b1aafSjsg void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
228522873acaSjsg {
22861bb76ff1Sjsg if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) ||
22871bb76ff1Sjsg (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1))) {
2288ad8b1aafSjsg WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
2289ad8b1aafSjsg WARN_ON(adev->gmc.sdpif_register !=
2290ad8b1aafSjsg RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
2291ad8b1aafSjsg }
229222873acaSjsg }
229322873acaSjsg
229422873acaSjsg /**
2295fb4d8502Sjsg * gmc_v9_0_gart_enable - gart enable
2296fb4d8502Sjsg *
2297fb4d8502Sjsg * @adev: amdgpu_device pointer
2298fb4d8502Sjsg */
gmc_v9_0_gart_enable(struct amdgpu_device * adev)2299fb4d8502Sjsg static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
2300fb4d8502Sjsg {
2301fb4d8502Sjsg int r;
2302fb4d8502Sjsg
23035ca02815Sjsg if (adev->gmc.xgmi.connected_to_cpu)
23045ca02815Sjsg amdgpu_gmc_init_pdb0(adev);
23055ca02815Sjsg
2306c349dbc7Sjsg if (adev->gart.bo == NULL) {
2307fb4d8502Sjsg dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
2308fb4d8502Sjsg return -EINVAL;
2309fb4d8502Sjsg }
23105ca02815Sjsg
23111bb76ff1Sjsg amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
2312f005ef32Sjsg
2313f005ef32Sjsg if (!adev->in_s0ix) {
2314ad8b1aafSjsg r = adev->gfxhub.funcs->gart_enable(adev);
2315fb4d8502Sjsg if (r)
2316fb4d8502Sjsg return r;
2317f005ef32Sjsg }
2318fb4d8502Sjsg
2319ad8b1aafSjsg r = adev->mmhub.funcs->gart_enable(adev);
2320fb4d8502Sjsg if (r)
2321fb4d8502Sjsg return r;
2322fb4d8502Sjsg
23235ca02815Sjsg DRM_INFO("PCIE GART of %uM enabled.\n",
2324f005ef32Sjsg (unsigned int)(adev->gmc.gart_size >> 20));
23255ca02815Sjsg if (adev->gmc.pdb0_bo)
23265ca02815Sjsg DRM_INFO("PDB0 located at 0x%016llX\n",
23275ca02815Sjsg (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
23285ca02815Sjsg DRM_INFO("PTB located at 0x%016llX\n",
2329c349dbc7Sjsg (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
23305ca02815Sjsg
2331c349dbc7Sjsg return 0;
2332c349dbc7Sjsg }
2333c349dbc7Sjsg
gmc_v9_0_hw_init(void * handle)2334c349dbc7Sjsg static int gmc_v9_0_hw_init(void *handle)
2335c349dbc7Sjsg {
2336c349dbc7Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2337c349dbc7Sjsg bool value;
23381bb76ff1Sjsg int i, r;
2339c349dbc7Sjsg
2340c349dbc7Sjsg /* The sequence of these two function calls matters.*/
2341c349dbc7Sjsg gmc_v9_0_init_golden_registers(adev);
2342c349dbc7Sjsg
2343c349dbc7Sjsg if (adev->mode_info.num_crtc) {
2344c349dbc7Sjsg /* Lockout access through VGA aperture*/
2345c349dbc7Sjsg WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
2346c349dbc7Sjsg /* disable VGA render */
2347c349dbc7Sjsg WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
2348c349dbc7Sjsg }
2349c349dbc7Sjsg
2350ad8b1aafSjsg if (adev->mmhub.funcs->update_power_gating)
2351ad8b1aafSjsg adev->mmhub.funcs->update_power_gating(adev, true);
2352ad8b1aafSjsg
23535ca02815Sjsg adev->hdp.funcs->init_registers(adev);
2354afca2431Sjsg
2355fb4d8502Sjsg /* After HDP is initialized, flush HDP.*/
23565ca02815Sjsg adev->hdp.funcs->flush_hdp(adev, NULL);
2357fb4d8502Sjsg
2358fb4d8502Sjsg if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
2359fb4d8502Sjsg value = false;
2360fb4d8502Sjsg else
2361fb4d8502Sjsg value = true;
2362fb4d8502Sjsg
2363c349dbc7Sjsg if (!amdgpu_sriov_vf(adev)) {
2364f005ef32Sjsg if (!adev->in_s0ix)
2365ad8b1aafSjsg adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2366ad8b1aafSjsg adev->mmhub.funcs->set_fault_enable_default(adev, value);
2367fb4d8502Sjsg }
2368f005ef32Sjsg for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
2369f005ef32Sjsg if (adev->in_s0ix && (i == AMDGPU_GFXHUB(0)))
2370f005ef32Sjsg continue;
2371c349dbc7Sjsg gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
2372f005ef32Sjsg }
2373fb4d8502Sjsg
2374c349dbc7Sjsg if (adev->umc.funcs && adev->umc.funcs->init_registers)
2375c349dbc7Sjsg adev->umc.funcs->init_registers(adev);
2376fb4d8502Sjsg
2377fb4d8502Sjsg r = gmc_v9_0_gart_enable(adev);
23781bb76ff1Sjsg if (r)
23791bb76ff1Sjsg return r;
2380fb4d8502Sjsg
23811bb76ff1Sjsg if (amdgpu_emu_mode == 1)
23821bb76ff1Sjsg return amdgpu_gmc_vram_checking(adev);
2383*c606d7e3Sjsg
2384*c606d7e3Sjsg return 0;
2385fb4d8502Sjsg }
2386fb4d8502Sjsg
2387fb4d8502Sjsg /**
2388fb4d8502Sjsg * gmc_v9_0_gart_disable - gart disable
2389fb4d8502Sjsg *
2390fb4d8502Sjsg * @adev: amdgpu_device pointer
2391fb4d8502Sjsg *
2392fb4d8502Sjsg * This disables all VM page table.
2393fb4d8502Sjsg */
gmc_v9_0_gart_disable(struct amdgpu_device * adev)2394fb4d8502Sjsg static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
2395fb4d8502Sjsg {
2396f005ef32Sjsg if (!adev->in_s0ix)
2397ad8b1aafSjsg adev->gfxhub.funcs->gart_disable(adev);
2398ad8b1aafSjsg adev->mmhub.funcs->gart_disable(adev);
2399fb4d8502Sjsg }
2400fb4d8502Sjsg
gmc_v9_0_hw_fini(void * handle)2401fb4d8502Sjsg static int gmc_v9_0_hw_fini(void *handle)
2402fb4d8502Sjsg {
2403fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2404fb4d8502Sjsg
24053dfc65b6Sjsg gmc_v9_0_gart_disable(adev);
24063dfc65b6Sjsg
2407fb4d8502Sjsg if (amdgpu_sriov_vf(adev)) {
2408fb4d8502Sjsg /* full access mode, so don't touch any GMC register */
2409fb4d8502Sjsg DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
2410fb4d8502Sjsg return 0;
2411fb4d8502Sjsg }
2412fb4d8502Sjsg
24131bb76ff1Sjsg /*
24141bb76ff1Sjsg * Pair the operations did in gmc_v9_0_hw_init and thus maintain
24151bb76ff1Sjsg * a correct cached state for GMC. Otherwise, the "gate" again
24161bb76ff1Sjsg * operation on S3 resuming will fail due to wrong cached state.
24171bb76ff1Sjsg */
24181bb76ff1Sjsg if (adev->mmhub.funcs->update_power_gating)
24191bb76ff1Sjsg adev->mmhub.funcs->update_power_gating(adev, false);
24201bb76ff1Sjsg
2421fb4d8502Sjsg amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
2422fb4d8502Sjsg
242393e84eb6Sjsg if (adev->gmc.ecc_irq.funcs &&
242493e84eb6Sjsg amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))
242593e84eb6Sjsg amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
242693e84eb6Sjsg
2427fb4d8502Sjsg return 0;
2428fb4d8502Sjsg }
2429fb4d8502Sjsg
gmc_v9_0_suspend(void * handle)2430fb4d8502Sjsg static int gmc_v9_0_suspend(void *handle)
2431fb4d8502Sjsg {
2432fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2433fb4d8502Sjsg
2434ad8b1aafSjsg return gmc_v9_0_hw_fini(adev);
2435fb4d8502Sjsg }
2436fb4d8502Sjsg
gmc_v9_0_resume(void * handle)2437fb4d8502Sjsg static int gmc_v9_0_resume(void *handle)
2438fb4d8502Sjsg {
2439fb4d8502Sjsg int r;
2440fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2441fb4d8502Sjsg
2442fb4d8502Sjsg r = gmc_v9_0_hw_init(adev);
2443fb4d8502Sjsg if (r)
2444fb4d8502Sjsg return r;
2445fb4d8502Sjsg
2446fb4d8502Sjsg amdgpu_vmid_reset_all(adev);
2447fb4d8502Sjsg
2448fb4d8502Sjsg return 0;
2449fb4d8502Sjsg }
2450fb4d8502Sjsg
gmc_v9_0_is_idle(void * handle)2451fb4d8502Sjsg static bool gmc_v9_0_is_idle(void *handle)
2452fb4d8502Sjsg {
2453fb4d8502Sjsg /* MC is always ready in GMC v9.*/
2454fb4d8502Sjsg return true;
2455fb4d8502Sjsg }
2456fb4d8502Sjsg
gmc_v9_0_wait_for_idle(void * handle)2457fb4d8502Sjsg static int gmc_v9_0_wait_for_idle(void *handle)
2458fb4d8502Sjsg {
2459fb4d8502Sjsg /* There is no need to wait for MC idle in GMC v9.*/
2460fb4d8502Sjsg return 0;
2461fb4d8502Sjsg }
2462fb4d8502Sjsg
gmc_v9_0_soft_reset(void * handle)2463fb4d8502Sjsg static int gmc_v9_0_soft_reset(void *handle)
2464fb4d8502Sjsg {
2465fb4d8502Sjsg /* XXX for emulation.*/
2466fb4d8502Sjsg return 0;
2467fb4d8502Sjsg }
2468fb4d8502Sjsg
gmc_v9_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)2469fb4d8502Sjsg static int gmc_v9_0_set_clockgating_state(void *handle,
2470fb4d8502Sjsg enum amd_clockgating_state state)
2471fb4d8502Sjsg {
2472fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2473fb4d8502Sjsg
2474ad8b1aafSjsg adev->mmhub.funcs->set_clockgating(adev, state);
2475c349dbc7Sjsg
2476c349dbc7Sjsg athub_v1_0_set_clockgating(adev, state);
2477c349dbc7Sjsg
2478c349dbc7Sjsg return 0;
2479fb4d8502Sjsg }
2480fb4d8502Sjsg
gmc_v9_0_get_clockgating_state(void * handle,u64 * flags)24811bb76ff1Sjsg static void gmc_v9_0_get_clockgating_state(void *handle, u64 *flags)
2482fb4d8502Sjsg {
2483fb4d8502Sjsg struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2484fb4d8502Sjsg
2485ad8b1aafSjsg adev->mmhub.funcs->get_clockgating(adev, flags);
2486c349dbc7Sjsg
2487c349dbc7Sjsg athub_v1_0_get_clockgating(adev, flags);
2488fb4d8502Sjsg }
2489fb4d8502Sjsg
gmc_v9_0_set_powergating_state(void * handle,enum amd_powergating_state state)2490fb4d8502Sjsg static int gmc_v9_0_set_powergating_state(void *handle,
2491fb4d8502Sjsg enum amd_powergating_state state)
2492fb4d8502Sjsg {
2493fb4d8502Sjsg return 0;
2494fb4d8502Sjsg }
2495fb4d8502Sjsg
2496fb4d8502Sjsg const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
2497fb4d8502Sjsg .name = "gmc_v9_0",
2498fb4d8502Sjsg .early_init = gmc_v9_0_early_init,
2499fb4d8502Sjsg .late_init = gmc_v9_0_late_init,
2500fb4d8502Sjsg .sw_init = gmc_v9_0_sw_init,
2501fb4d8502Sjsg .sw_fini = gmc_v9_0_sw_fini,
2502fb4d8502Sjsg .hw_init = gmc_v9_0_hw_init,
2503fb4d8502Sjsg .hw_fini = gmc_v9_0_hw_fini,
2504fb4d8502Sjsg .suspend = gmc_v9_0_suspend,
2505fb4d8502Sjsg .resume = gmc_v9_0_resume,
2506fb4d8502Sjsg .is_idle = gmc_v9_0_is_idle,
2507fb4d8502Sjsg .wait_for_idle = gmc_v9_0_wait_for_idle,
2508fb4d8502Sjsg .soft_reset = gmc_v9_0_soft_reset,
2509fb4d8502Sjsg .set_clockgating_state = gmc_v9_0_set_clockgating_state,
2510fb4d8502Sjsg .set_powergating_state = gmc_v9_0_set_powergating_state,
2511fb4d8502Sjsg .get_clockgating_state = gmc_v9_0_get_clockgating_state,
2512fb4d8502Sjsg };
2513fb4d8502Sjsg
2514f005ef32Sjsg const struct amdgpu_ip_block_version gmc_v9_0_ip_block = {
2515fb4d8502Sjsg .type = AMD_IP_BLOCK_TYPE_GMC,
2516fb4d8502Sjsg .major = 9,
2517fb4d8502Sjsg .minor = 0,
2518fb4d8502Sjsg .rev = 0,
2519fb4d8502Sjsg .funcs = &gmc_v9_0_ip_funcs,
2520fb4d8502Sjsg };
2521