1 /*==============================================================================
2 Copyright(c) 2017 Intel Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a
5 copy of this software and associated documentation files(the "Software"),
6 to deal in the Software without restriction, including without limitation
7 the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and / or sell copies of the Software, and to permit persons to whom the
9 Software is furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included
12 in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 OTHER DEALINGS IN THE SOFTWARE.
21 ============================================================================*/
22
23 #include "Internal/Common/GmmLibInc.h"
24 #include "External/Common/GmmCachePolicy.h"
25
26 //=============================================================================
27 //
28 // Function: __GmmGen9InitCachePolicy
29 //
30 // Desc: This function initializes the cache policy
31 //
32 // Parameters: pCachePolicy -> Ptr to array to be populated with the
33 // mapping of usages -> cache settings.
34 //
35 // Return: GMM_STATUS
36 //
37 //-----------------------------------------------------------------------------
InitCachePolicy()38 GMM_STATUS GmmLib::GmmGen9CachePolicy::InitCachePolicy()
39 {
40 __GMM_ASSERTPTR(pCachePolicy, GMM_ERROR);
41
42 #if defined(GMM_DYNAMIC_MOCS_TABLE)
43 #define DEFINE_CACHE_ELEMENT(usage, llc, ellc, l3, age, i915) DEFINE_CP_ELEMENT(usage, llc, ellc, l3, 0, age, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
44 #else
45 // i915 only supports three GEN9 MOCS entires:
46 // MOCS[0]...LLC=0, ELLC=0, L3=0, AGE=0
47 // MOCS[1]...<N/A for GmmLib Purposes>
48 // MOCS[2]...LLC=1, ELLC=1, L3=1, AGE=3
49 #define DEFINE_CACHE_ELEMENT(usage, llc, ellc, l3, age, i915) \
50 do \
51 { \
52 if((i915) == 0) \
53 { \
54 DEFINE_CP_ELEMENT(usage, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);\
55 } \
56 else if((i915) == 2) \
57 { \
58 DEFINE_CP_ELEMENT(usage, 1, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);\
59 } \
60 else \
61 { \
62 GMM_ASSERTDPF(0, "Invalid i915 MOCS specified"); \
63 } \
64 } while(0) ////////////////////////////////////////////////////////////////
65 #endif
66 #include "GmmGen9CachePolicy.h"
67
68 #define TC_LLC (1)
69 #define TC_ELLC (0)
70 #define TC_LLC_ELLC (2)
71
72 #define LeCC_UNCACHEABLE (0x1)
73 #define LeCC_WB_CACHEABLE (0x3)
74
75 #define L3_UNCACHEABLE (0x1)
76 #define L3_WB_CACHEABLE (0x3)
77
78 #define DISABLE_SKIP_CACHING_CONTROL (0x0)
79 #define ENABLE_SKIP_CACHING_CONTROL (0x1)
80
81 {
82 uint32_t CurrentMaxIndex = 0;
83 GMM_CACHE_POLICY_TBL_ELEMENT *pCachePolicyTblElement = pGmmLibContext->GetCachePolicyTlbElement();
84
85 bool LLC = (pGmmLibContext->GetGtSysInfo()->LLCCacheSizeInKb > 0); // aka "Core -vs- Atom".
86
87 #if defined(_WIN32)
88 {
89 pCachePolicyTblElement[0].L3.Cacheability = L3_UNCACHEABLE;
90 pCachePolicyTblElement[0].LeCC.Cacheability = LeCC_UNCACHEABLE;
91 pCachePolicyTblElement[0].LeCC.TargetCache = LLC ? TC_LLC_ELLC : TC_ELLC; // No LLC for Broxton, GLK - keep clear configuration for LLC
92 }
93 #else
94 {
95 #define I915_GEN9_MOCS_ENTRIES 3
96 GMM_CACHE_POLICY_TBL_ELEMENT *pEntry = pCachePolicyTblElement;
97 C_ASSERT(I915_GEN9_MOCS_ENTRIES <= GMM_GEN9_MAX_NUMBER_MOCS_INDEXES);
98
99 // I915_MOCS_UNCACHED(0)...
100 pEntry[0].L3.Cacheability = L3_UNCACHEABLE;
101 pEntry[0].LeCC.Cacheability = LeCC_UNCACHEABLE;
102 pEntry[0].LeCC.TargetCache = TC_LLC_ELLC;
103
104 // I915_MOCS_PTE(1)...
105 pEntry[1] = pEntry[0]; // Unused by GmmLib clients, so set to UC.
106 CurrentMaxIndex++;
107
108 // I915_MOCS_CACHED(2)...
109 pEntry[2].L3.Cacheability = L3_WB_CACHEABLE;
110 pEntry[2].LeCC.Cacheability = LLC ? LeCC_WB_CACHEABLE : LeCC_UNCACHEABLE;
111 pEntry[2].LeCC.TargetCache = TC_LLC_ELLC;
112 pEntry[2].LeCC.LRUM = 3;
113 CurrentMaxIndex++;
114 }
115 #endif
116
117 // Process the cache policy and fill in the look up table
118 for(uint32_t Usage = 0; Usage < GMM_RESOURCE_USAGE_MAX; Usage++)
119 {
120 bool CachePolicyError = false;
121 uint64_t PTEValue = 0;
122 int32_t CPTblIdx = -1;
123 uint32_t j = 0;
124 GMM_CACHE_POLICY_TBL_ELEMENT UsageEle = {0};
125 UsageEle.LeCC.Reserved = 0; // Reserved bits zeroe'd, this is so we
126 // we can compare the unioned LeCC.DwordValue.
127 UsageEle.LeCC.SCF = pCachePolicy[Usage].SCF;
128 UsageEle.LeCC.PFM = 0; // TODO: decide what the page faulting mode should be
129 UsageEle.LeCC.SCC = 0;
130 UsageEle.LeCC.ESC = 0;
131 if(pCachePolicy[Usage].LeCC_SCC)
132 {
133 UsageEle.LeCC.SCC = pCachePolicy[Usage].LeCC_SCC;
134 UsageEle.LeCC.ESC = ENABLE_SKIP_CACHING_CONTROL;
135 }
136 UsageEle.LeCC.AOM = pCachePolicy[Usage].AOM;
137 UsageEle.LeCC.LRUM = pCachePolicy[Usage].AGE;
138
139 // default to LLC/ELLC target cache.
140 UsageEle.LeCC.TargetCache = TC_LLC_ELLC;
141 UsageEle.LeCC.Cacheability = LeCC_WB_CACHEABLE;
142
143 if(pGmmLibContext->GetPlatformInfo().Platform.eProductFamily == IGFX_BROXTON ||
144 pGmmLibContext->GetPlatformInfo().Platform.eProductFamily == IGFX_GEMINILAKE)
145 {
146 UsageEle.LeCC.AOM = 0;
147 UsageEle.LeCC.Cacheability = LeCC_UNCACHEABLE; // To avoid side effects use 01b even though 01b(UC) 11b(WB) are equivalent option
148
149 #if defined(GMM_DYNAMIC_MOCS_TABLE)
150 UsageEle.LeCC.TargetCache = TC_LLC; // No LLC for Broxton, but we still set it to LLC since it is needed for IA coherency cases
151 UsageEle.LeCC.LRUM = 0;
152 #else
153 UsageEle.LeCC.TargetCache = TC_LLC_ELLC; // To match I915_GEN9_MOCS[0]
154 #endif
155 }
156 else
157 {
158 if(pCachePolicy[Usage].LLC && pCachePolicy[Usage].ELLC)
159 {
160 UsageEle.LeCC.TargetCache = TC_LLC_ELLC;
161 }
162 else if(pCachePolicy[Usage].LLC)
163 {
164 UsageEle.LeCC.TargetCache = TC_LLC;
165 }
166 else if(pCachePolicy[Usage].ELLC)
167 {
168 UsageEle.LeCC.TargetCache = TC_ELLC;
169 }
170 else
171 {
172 UsageEle.LeCC.Cacheability = LeCC_UNCACHEABLE;
173 }
174 }
175
176 UsageEle.L3.Reserved = 0; // Reserved bits zeroe'd, this is so we
177 // we can compare the unioned L3.UshortValue.
178 UsageEle.L3.ESC = DISABLE_SKIP_CACHING_CONTROL;
179 UsageEle.L3.SCC = 0;
180 UsageEle.L3.Cacheability = pCachePolicy[Usage].L3 ? L3_WB_CACHEABLE : L3_UNCACHEABLE;
181 if(pCachePolicy[Usage].L3_SCC)
182 {
183 UsageEle.L3.ESC = ENABLE_SKIP_CACHING_CONTROL;
184 UsageEle.L3.SCC = (uint16_t)pCachePolicy[Usage].L3_SCC;
185 }
186 for(j = 0; j <= CurrentMaxIndex; j++)
187 {
188 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &pCachePolicyTblElement[j];
189 if(TblEle->LeCC.DwordValue == UsageEle.LeCC.DwordValue &&
190 TblEle->L3.UshortValue == UsageEle.L3.UshortValue)
191 {
192 CPTblIdx = j;
193 break;
194 }
195 }
196
197 // Didn't find the caching settings in one of the already programmed lookup table entries.
198 // Need to add a new lookup table entry.
199 if(CPTblIdx == -1)
200 {
201 if(CurrentMaxIndex < GMM_GEN9_MAX_NUMBER_MOCS_INDEXES - 1)
202 {
203 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &(pCachePolicyTblElement[++CurrentMaxIndex]);
204 CPTblIdx = CurrentMaxIndex;
205
206 TblEle->LeCC.DwordValue = UsageEle.LeCC.DwordValue;
207 TblEle->L3.UshortValue = UsageEle.L3.UshortValue;
208 }
209 else
210 {
211 // Too many unique caching combinations to program the
212 // MOCS lookup table.
213 CachePolicyError = true;
214 GMM_ASSERTDPF(
215 "Cache Policy Init Error: Invalid Cache Programming, too many unique caching combinations"
216 "(we only support GMM_GEN_MAX_NUMBER_MOCS_INDEXES = %d)",
217 GMM_GEN9_MAX_NUMBER_MOCS_INDEXES);
218 // Set cache policy index to uncached.
219 CPTblIdx = 0;
220 }
221 }
222
223 if(!GetUsagePTEValue(pCachePolicy[Usage], Usage, &PTEValue))
224 {
225 CachePolicyError = true;
226 }
227
228 pCachePolicy[Usage].PTE.DwordValue = PTEValue & 0xFFFFFFFF;
229 pCachePolicy[Usage].PTE.HighDwordValue = 0;
230 pCachePolicy[Usage].MemoryObjectOverride.Gen9.Index = CPTblIdx;
231
232 pCachePolicy[Usage].Override = ALWAYS_OVERRIDE;
233
234 if(CachePolicyError)
235 {
236 GMM_ASSERTDPF("Cache Policy Init Error: Invalid Cache Programming - Element %d", Usage);
237 }
238 }
239 CurrentMaxMocsIndex = CurrentMaxIndex;
240 CurrentMaxL1HdcMocsIndex = 0;
241 }
242
243 return GMM_SUCCESS;
244 }
245
246 /////////////////////////////////////////////////////////////////////////////////////
247 /// Initializes the Gfx PAT tables for AdvCtx and Gfx MMIO/Private PAT
248 /// PAT0 = WB_COHERENT or UC depending on WaGttPat0WB
249 /// PAT1 = UC or WB_COHERENT depending on WaGttPat0WB
250 /// PAT2 = WB_MOCSLESS, with TC = eLLC+LLC
251 /// PAT3 = WB
252 /// PAT4 = WT
253 /// PAT5 = WC
254 /// PAT6 = WC
255 /// PAT7 = WC
256 /// HLD says to set to PAT0/1 to WC, but since we don't have a WC in GPU,
257 /// WC option is same as UC. Hence setting PAT0 or PAT1 to UC.
258 /// Unused PAT's (5,6,7) are set to WC.
259 ///
260 /// @return GMM_STATUS
261 /////////////////////////////////////////////////////////////////////////////////////
SetupPAT()262 GMM_STATUS GmmLib::GmmGen9CachePolicy::SetupPAT()
263 {
264 GMM_STATUS Status = GMM_SUCCESS;
265 #if(defined(__GMM_KMD__))
266 uint32_t i = 0;
267
268 GMM_GFX_MEMORY_TYPE GfxMemType = GMM_GFX_UC_WITH_FENCE;
269 // No optional selection on Age or Target Cache because for an SVM-OS Age and
270 // Target Cache would not work [for an SVM-OS the Page Table is shared with IA
271 // and we don't have control of the PAT Idx]. If there is a strong ask from D3D
272 // or the performance analysis team, Age could be added.
273 // Add Class of Service when required.
274 GMM_GFX_TARGET_CACHE GfxTargetCache = GMM_GFX_TC_ELLC_LLC;
275 uint8_t Age = 1;
276 uint8_t ServiceClass = 0;
277 int32_t * pPrivatePATTableMemoryType = NULL;
278
279 pPrivatePATTableMemoryType = pGmmLibContext->GetPrivatePATTableMemoryType();
280
281 __GMM_ASSERT(pGmmLibContext->GetSkuTable().FtrIA32eGfxPTEs);
282
283 for(i = 0; i < GMM_NUM_GFX_PAT_TYPES; i++)
284 {
285 pPrivatePATTableMemoryType[i] = -1;
286 }
287
288 // Set values for GmmGlobalInfo PrivatePATTable
289 for(i = 0; i < GMM_NUM_PAT_ENTRIES; i++)
290 {
291 GMM_PRIVATE_PAT PAT = {0};
292
293 if(pGmmLibContext->GetSkuTable().FtrMemTypeMocsDeferPAT)
294 {
295 GfxTargetCache = GMM_GFX_TC_ELLC_ONLY;
296 }
297 else
298 {
299 GfxTargetCache = GMM_GFX_TC_ELLC_LLC;
300 }
301
302 switch(i)
303 {
304 case PAT0:
305 if(pGmmLibContext->GetWaTable().WaGttPat0)
306 {
307 if(pGmmLibContext->GetWaTable().WaGttPat0WB)
308 {
309 GfxMemType = GMM_GFX_WB;
310 if(GFX_IS_ATOM_PLATFORM(pGmmLibContext))
311 {
312 PAT.PreGen10.Snoop = 1;
313 }
314 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
315 }
316 else
317 {
318 GfxMemType = GMM_GFX_UC_WITH_FENCE;
319 pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT0;
320 }
321 }
322 else // if GTT is not tied to PAT0 then WaGttPat0WB is NA
323 {
324 GfxMemType = GMM_GFX_WB;
325 if(GFX_IS_ATOM_PLATFORM(pGmmLibContext))
326 {
327 PAT.PreGen10.Snoop = 1;
328 }
329 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
330 }
331 break;
332
333 case PAT1:
334 if(pGmmLibContext->GetWaTable().WaGttPat0 && !pGmmLibContext->GetWaTable().WaGttPat0WB)
335 {
336 GfxMemType = GMM_GFX_WB;
337 if(GFX_IS_ATOM_PLATFORM(pGmmLibContext))
338 {
339 PAT.PreGen10.Snoop = 1;
340 }
341 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT1;
342 }
343 else
344 {
345 GfxMemType = GMM_GFX_UC_WITH_FENCE;
346 pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT1;
347 }
348 break;
349
350 case PAT2:
351 // This PAT idx shall be used for MOCS'Less resources like Page Tables
352 // Page Tables have TC hardcoded to eLLC+LLC in Adv Ctxt. Hence making this to have same in Leg Ctxt.
353 // For BDW-H, due to Perf issue, TC has to be eLLC only for Page Tables when eDRAM is present.
354 GfxMemType = GMM_GFX_WB;
355 GfxTargetCache = GMM_GFX_TC_ELLC_LLC;
356 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_MOCSLESS] = PAT2;
357 break;
358
359 case PAT3:
360 GfxMemType = GMM_GFX_WB;
361 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB] = PAT3;
362 break;
363
364 case PAT4:
365 GfxMemType = GMM_GFX_WT;
366 pPrivatePATTableMemoryType[GMM_GFX_PAT_WT] = PAT4;
367 break;
368
369 case PAT5:
370 case PAT6:
371 case PAT7:
372 GfxMemType = GMM_GFX_WC;
373 pPrivatePATTableMemoryType[GMM_GFX_PAT_WC] = PAT5;
374 break;
375
376 default:
377 __GMM_ASSERT(0);
378 Status = GMM_ERROR;
379 }
380
381 PAT.PreGen10.MemoryType = GfxMemType;
382 PAT.PreGen10.TargetCache = GfxTargetCache;
383 PAT.PreGen10.Age = Age;
384
385 SetPrivatePATEntry(i, PAT);
386 }
387
388 #else
389 Status = GMM_ERROR;
390 #endif
391 return Status;
392 }
393