1 /*============================================================================== 2 Copyright(c) 2017 Intel Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a 5 copy of this software and associated documentation files(the "Software"), 6 to deal in the Software without restriction, including without limitation 7 the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 and / or sell copies of the Software, and to permit persons to whom the 9 Software is furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be included 12 in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 OTHER DEALINGS IN THE SOFTWARE. 21 ============================================================================*/ 22 #pragma once 23 24 #ifdef __cplusplus 25 extern "C" { 26 #endif 27 #include "GmmCachePolicyExt.h" 28 29 #define ALWAYS_OVERRIDE 0xffffffff 30 #define NEVER_OVERRIDE 0x0 31 32 // IA PAT MSR Register 33 #define GMM_IA32e_PAT_MSR 0x277 34 35 #define GMM_PAT_ERROR ((uint32_t)-1) 36 37 typedef struct GMM_CACHE_POLICY_ELEMENT_REC 38 { 39 uint32_t IDCode; 40 union { 41 struct{ 42 uint64_t LLC : 1; 43 uint64_t ELLC : 1; 44 uint64_t L3 : 1; 45 uint64_t WT : 1; 46 uint64_t AGE : 2; 47 uint64_t AOM : 1; 48 uint64_t LeCC_SCC : 3; 49 uint64_t L3_SCC : 3; 50 uint64_t SCF : 1; // Snoop Control Field for BXT 51 uint64_t CoS : 2; // Class of Service, driver default to Class 0 52 uint64_t SSO : 2; // Self Snoop Override control and value 53 uint64_t HDCL1 : 1; // HDC L1 caching enable/disable 54 uint64_t L3Eviction : 2; // Specify L3-eviction type (NA, ReadOnly, Standard, Special) 55 uint64_t SegOv : 3; // Override seg-pref (none, local-only, sys-only, etc) 56 uint64_t GlbGo : 1; // Global GO point - L3 or Memory 57 uint64_t UcLookup : 1; // Snoop L3 for uncached 58 uint64_t L1CC : 3; // L1 Cache Control 59 uint64_t Initialized : 1; 60 uint64_t Reserved : 34; 61 62 }; 63 uint64_t Value; 64 }; 65 66 MEMORY_OBJECT_CONTROL_STATE MemoryObjectOverride; 67 MEMORY_OBJECT_CONTROL_STATE MemoryObjectNoOverride; 68 GMM_PTE_CACHE_CONTROL_BITS PTE; 69 uint32_t Override; 70 uint32_t IsOverridenByRegkey; // Flag to indicate If usage settings are overridden by regkey 71 }GMM_CACHE_POLICY_ELEMENT; 72 73 // One entry in the SKL/CNL cache lookup table 74 typedef struct GMM_CACHE_POLICY_TBL_ELEMENT_REC { 75 union { 76 struct 77 { 78 uint32_t Cacheability : 2; // LLC/eDRAM Cacheability Control (LeCC). 79 uint32_t TargetCache : 2; // Target Cache (TC). 80 uint32_t LRUM : 2; // LRU (Cache Replacement) Management (LRUM). 81 uint32_t AOM : 1; // Do Not Allocate on Miss (AOM) - Not used 82 uint32_t ESC : 1; // Enable Skip Caching (ESC) on LLC 83 uint32_t SCC : 3; // Skip Caching Control (SCC). 84 uint32_t PFM : 3; // Page Fault Mode (PFM) 85 uint32_t SCF : 1; // Snoop Control Field (SCF) 86 uint32_t CoS : 2; // Class of Service (COS)- CNL+ 87 uint32_t SelfSnoop : 2; // Self Snoop override or not MIDI settings - CNL+ 88 uint32_t Reserved : 13; 89 } ; 90 uint32_t DwordValue; 91 } LeCC; 92 93 union { 94 struct 95 { 96 uint16_t ESC : 1; // Enable Skip Caching (ESC) for L3. 97 uint16_t SCC : 3; // Skip Caching Control (SCC) for L3. 98 uint16_t Cacheability : 2; // L3 Cacheability Control (L3CC). 99 uint16_t GlobalGo : 1; // Global Go (GLBGO). 100 uint16_t UCLookup : 1; // UC L3 Lookup (UcL3Lookup). 101 uint16_t Reserved : 8; 102 } ; 103 uint16_t UshortValue; 104 } L3; 105 uint8_t HDCL1; 106 } GMM_CACHE_POLICY_TBL_ELEMENT; 107 108 typedef enum GMM_IA32e_MEMORY_TYPE_REC 109 { 110 GMM_IA32e_UC = 0x0, 111 GMM_IA32e_WC = 0x1, 112 GMM_IA32e_WT = 0x4, 113 GMM_IA32e_WP = 0x5, 114 GMM_IA32e_WB = 0x6, 115 GMM_IA32e_UCMINUS = 0x7 116 } GMM_IA32e_MEMORY_TYPE; 117 118 typedef enum GMM_GFX_MEMORY_TYPE_REC 119 { 120 GMM_GFX_UC_WITH_FENCE = 0x0, 121 GMM_GFX_WC = 0x1, 122 GMM_GFX_WT = 0x2, 123 GMM_GFX_WB = 0x3 124 } GMM_GFX_MEMORY_TYPE; 125 126 typedef enum GMM_GFX_PAT_TYPE_REC 127 { 128 GMM_GFX_PAT_WB_COHERENT = 0x0, // WB + Snoop : Atom 129 GMM_GFX_PAT_WC = 0x1, 130 GMM_GFX_PAT_WB_MOCSLESS = 0x2, // WB + eLLC/LLC : MOCSLess Resource (PageTables) 131 GMM_GFX_PAT_WB = 0x3, 132 GMM_GFX_PAT_WT = 0x4, 133 GMM_GFX_PAT_UC = 0x5 134 } GMM_GFX_PAT_TYPE; 135 136 typedef enum GMM_GFX_PAT_IDX_REC 137 { 138 PAT0 = 0x0, // Will be tied to GMM_GFX_PAT_WB_COHERENT or GMM_GFX_PAT_UC depending on WaGttPat0WB 139 PAT1, // Will be tied to GMM_GFX_PAT_UC or GMM_GFX_PAT_WB_COHERENT depending on WaGttPat0WB 140 PAT2, // Will be tied to GMM_GFX_PAT_WB_MOCSLESS 141 PAT3, // Will be tied to GMM_GFX_PAT_WB 142 PAT4, // Will be tied to GMM_GFX_PAT_WT 143 PAT5, // Will be tied to GMM_GFX_PAT_WC 144 PAT6, // Will be tied to GMM_GFX_PAT_WC 145 PAT7 // Will be tied to GMM_GFX_PAT_WC 146 }GMM_GFX_PAT_IDX; 147 148 #define GFX_IS_ATOM_PLATFORM(pGmmLibContext) (GmmGetSkuTable(pGmmLibContext)->FtrLCIA) 149 150 typedef enum GMM_GFX_TARGET_CACHE_REC 151 { 152 GMM_GFX_TC_ELLC_ONLY = 0x0, 153 GMM_GFX_TC_LLC_ONLY = 0x1, 154 GMM_GFX_TC_ELLC_LLC = 0x3 155 } GMM_GFX_TARGET_CACHE; 156 157 typedef union GMM_PRIVATE_PAT_REC { 158 159 struct 160 { 161 uint32_t MemoryType : 2; 162 uint32_t TargetCache : 2; 163 uint32_t Age : 2; 164 uint32_t Snoop : 1; 165 uint32_t Reserved : 1; 166 }PreGen10; 167 168 #if (IGFX_GEN >= IGFX_GEN10) 169 struct 170 { 171 uint32_t MemoryType : 2; 172 uint32_t TargetCache : 2; 173 uint32_t Age : 2; 174 uint32_t Reserved1 : 2; 175 uint32_t CoS : 2; 176 uint32_t Reserved2 : 22; 177 }Gen10; 178 #endif 179 180 struct 181 { 182 uint32_t MemoryType : 2; 183 uint32_t Reserved : 30; 184 }Gen12; 185 186 uint32_t Value; 187 188 } GMM_PRIVATE_PAT; 189 190 #ifdef _WIN32 191 192 #define GMM_CACHE_POLICY_OVERRIDE_REGISTY_PATH_REGISTRY_UMD "SOFTWARE\\Intel\\IGFX\\CachePolicy\\" 193 #define GMM_CACHE_POLICY_OVERRIDE_REGISTY_PATH_REGISTRY_KMD "\\REGISTRY\\MACHINE\\" GMM_CACHE_POLICY_OVERRIDE_REGISTY_PATH_REGISTRY_UMD 194 195 // No local variables or ternary (?:) operators should be defined in the 196 // REG_OVERRIDE macro block. This block will generate hundreds of instances 197 // of itself in the OverrideCachePolicy, each of which will geneate a 198 // copy of these local variables on the stack. The ternary operator will 199 // also generate local temparoary stack variables, and all these add up and 200 // can cause a stack overflow. 201 202 #define READOVERRIDES(Usage) \ 203 if (!REGISTRY_OVERRIDE_READ(Usage,LLC)) LLC = -1; \ 204 if (!REGISTRY_OVERRIDE_READ(Usage,ELLC)) ELLC = -1; \ 205 if (!REGISTRY_OVERRIDE_READ(Usage,L3)) L3 = -1; \ 206 if (!REGISTRY_OVERRIDE_READ(Usage,Age)) Age = -1; \ 207 if (!REGISTRY_OVERRIDE_READ(Usage,WT)) WT = -1; \ 208 if (!REGISTRY_OVERRIDE_READ(Usage,AOM)) AOM = -1; \ 209 if (!REGISTRY_OVERRIDE_READ(Usage,LeCC_SCC)) LeCC_SCC = -1; \ 210 if (!REGISTRY_OVERRIDE_READ(Usage,L3_SCC)) L3_SCC = -1; \ 211 if (!REGISTRY_OVERRIDE_READ(Usage,SCF)) SCF = -1; \ 212 if (!REGISTRY_OVERRIDE_READ(Usage,SSO)) SSO = -1; \ 213 if (!REGISTRY_OVERRIDE_READ(Usage,CoS)) CoS = -1; \ 214 if (!REGISTRY_OVERRIDE_READ(Usage,HDCL1)) HDCL1 = -1; \ 215 if (!REGISTRY_OVERRIDE_READ(Usage,L3Eviction)) L3Eviction = -1; \ 216 if (!REGISTRY_OVERRIDE_READ(Usage,GlbGo)) GlbGo = -1; \ 217 if (!REGISTRY_OVERRIDE_READ(Usage,UcLookup)) UcLookup = -1; \ 218 219 220 221 #define SETOVERRIDES(Usage) \ 222 if (LLC != -1) \ 223 { \ 224 pCachePolicy[Usage].LLC = LLC; \ 225 } \ 226 if (ELLC != -1) \ 227 { \ 228 pCachePolicy[Usage].ELLC = ELLC; \ 229 } \ 230 if (L3 != -1) \ 231 { \ 232 pCachePolicy[Usage].L3 = L3; \ 233 } \ 234 if (WT != -1) \ 235 { \ 236 pCachePolicy[Usage].WT = WT; \ 237 } \ 238 if (Age != -1) \ 239 { \ 240 pCachePolicy[Usage].AGE = Age; \ 241 } \ 242 if (AOM != -1) \ 243 { \ 244 pCachePolicy[Usage].AOM = AOM; \ 245 } \ 246 if (LeCC_SCC != -1) \ 247 { \ 248 pCachePolicy[Usage].LeCC_SCC = LeCC_SCC; \ 249 } \ 250 if (L3_SCC != -1) \ 251 { \ 252 pCachePolicy[Usage].L3_SCC = L3_SCC; \ 253 } \ 254 if (SCF != -1) \ 255 { \ 256 pCachePolicy[Usage].SCF = SCF; \ 257 } \ 258 if (SSO != -1) \ 259 { \ 260 pCachePolicy[Usage].SSO = SSO; \ 261 } \ 262 if (CoS != -1) \ 263 { \ 264 pCachePolicy[Usage].CoS = CoS; \ 265 } \ 266 if (HDCL1 != -1) \ 267 { \ 268 pCachePolicy[Usage].HDCL1 = HDCL1; \ 269 } \ 270 if (L3Eviction != -1) \ 271 { \ 272 pCachePolicy[Usage].L3Eviction = L3Eviction; \ 273 } \ 274 if (GlbGo != -1) \ 275 { \ 276 pCachePolicy[Usage].GlbGo = GlbGo; \ 277 } \ 278 if (UcLookup != -1) \ 279 { \ 280 pCachePolicy[Usage].UcLookup = UcLookup; \ 281 } \ 282 { \ 283 pCachePolicy[Usage].IsOverridenByRegkey = 1; \ 284 } 285 286 #ifdef __GMM_KMD__ 287 #define REG_OVERRIDE(Usage) \ 288 { \ 289 enum {IGNORED = 0, CURRENT, DEFAULT, UNCACHED}; \ 290 \ 291 REGISTRY_OVERRIDE_READ(Usage,Enable); \ 292 \ 293 if (Enable && (GenerateKeys != UNCACHED)) \ 294 { \ 295 READOVERRIDES(Usage) \ 296 } \ 297 \ 298 if (GenerateKeys == DEFAULT || (GenerateKeys == CURRENT && !Enable)) \ 299 { \ 300 REGISTRY_OVERRIDE_WRITE(Usage,LLC, pCachePolicy[Usage].LLC); \ 301 REGISTRY_OVERRIDE_WRITE(Usage,ELLC,pCachePolicy[Usage].ELLC); \ 302 REGISTRY_OVERRIDE_WRITE(Usage,L3, pCachePolicy[Usage].L3); \ 303 REGISTRY_OVERRIDE_WRITE(Usage,WT, pCachePolicy[Usage].WT); \ 304 REGISTRY_OVERRIDE_WRITE(Usage,Age, pCachePolicy[Usage].AGE); \ 305 REGISTRY_OVERRIDE_WRITE(Usage,AOM, pCachePolicy[Usage].AOM); \ 306 REGISTRY_OVERRIDE_WRITE(Usage,LeCC_SCC, pCachePolicy[Usage].LeCC_SCC); \ 307 REGISTRY_OVERRIDE_WRITE(Usage,L3_SCC, pCachePolicy[Usage].L3_SCC); \ 308 REGISTRY_OVERRIDE_WRITE(Usage,SCF, pCachePolicy[Usage].SCF); \ 309 REGISTRY_OVERRIDE_WRITE(Usage,SSO, pCachePolicy[Usage].SSO); \ 310 REGISTRY_OVERRIDE_WRITE(Usage,CoS, pCachePolicy[Usage].CoS); \ 311 REGISTRY_OVERRIDE_WRITE(Usage,HDCL1, pCachePolicy[Usage].HDCL1); \ 312 REGISTRY_OVERRIDE_WRITE(Usage,L3Eviction, pCachePolicy[Usage].L3Eviction); \ 313 REGISTRY_OVERRIDE_WRITE(Usage,Enable,0); \ 314 REGISTRY_OVERRIDE_WRITE(Usage,GlbGo, pCachePolicy[Usage].GlbGo); \ 315 REGISTRY_OVERRIDE_WRITE(Usage,UcLookup, pCachePolicy[Usage].UcLookup); \ 316 317 } \ 318 else if (GenerateKeys == UNCACHED || GenerateKeys == CURRENT) \ 319 { \ 320 if (GenerateKeys == UNCACHED) \ 321 { \ 322 Enable = 1; \ 323 } \ 324 REGISTRY_OVERRIDE_WRITE(Usage,LLC, LLC); \ 325 REGISTRY_OVERRIDE_WRITE(Usage,ELLC,ELLC); \ 326 REGISTRY_OVERRIDE_WRITE(Usage,L3, L3); \ 327 REGISTRY_OVERRIDE_WRITE(Usage,WT, WT); \ 328 REGISTRY_OVERRIDE_WRITE(Usage,Age, Age); \ 329 REGISTRY_OVERRIDE_WRITE(Usage,AOM, AOM); \ 330 REGISTRY_OVERRIDE_WRITE(Usage,LeCC_SCC, LeCC_SCC); \ 331 REGISTRY_OVERRIDE_WRITE(Usage,L3_SCC, L3_SCC); \ 332 REGISTRY_OVERRIDE_WRITE(Usage,SCF, SCF); \ 333 REGISTRY_OVERRIDE_WRITE(Usage,SSO, SSO); \ 334 REGISTRY_OVERRIDE_WRITE(Usage,CoS, CoS); \ 335 REGISTRY_OVERRIDE_WRITE(Usage,HDCL1, HDCL1); \ 336 REGISTRY_OVERRIDE_WRITE(Usage,L3Eviction, L3Eviction); \ 337 REGISTRY_OVERRIDE_WRITE(Usage,Enable,Enable); \ 338 REGISTRY_OVERRIDE_WRITE(Usage,GlbGo, GlbGo); \ 339 REGISTRY_OVERRIDE_WRITE(Usage,UcLookup, UcLookup); \ 340 } \ 341 \ 342 if (Enable) \ 343 { \ 344 SETOVERRIDES(Usage) \ 345 } \ 346 \ 347 UsageCount++; \ 348 } 349 #else 350 #define REG_OVERRIDE(Usage) \ 351 { \ 352 REGISTRY_OVERRIDE_READ(Usage,Enable); \ 353 if (Enable) \ 354 { \ 355 READOVERRIDES(Usage) \ 356 SETOVERRIDES(Usage) \ 357 } \ 358 UsageCount++; \ 359 } 360 #endif 361 362 #endif // #if _WIN32 363 364 #if __cplusplus 365 #include "GmmCachePolicyCommon.h" 366 #include "CachePolicy/GmmCachePolicyGen8.h" 367 #include "CachePolicy/GmmCachePolicyGen9.h" 368 #endif 369 370 // Function Prototypes 371 GMM_STATUS GmmInitializeCachePolicy(); 372 GMM_GFX_MEMORY_TYPE GmmGetWantedMemoryType(void *pLibContext, GMM_CACHE_POLICY_ELEMENT CachePolicy); 373 374 // Used for GMM ULT testing. 375 #ifdef __GMM_ULT 376 extern void __GmmULTCachePolicy(GMM_CACHE_POLICY_ELEMENT *pCachePolicy); 377 #endif 378 #ifdef __cplusplus 379 } 380 #endif 381