1 /*==============================================================================
2 Copyright(c) 2019 Intel Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a
5 copy of this software and associated documentation files(the "Software"),
6 to deal in the Software without restriction, including without limitation
7 the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and / or sell copies of the Software, and to permit persons to whom the
9 Software is furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included
12 in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 OTHER DEALINGS IN THE SOFTWARE.
21 ============================================================================*/
22
23
24 #include "Internal/Common/GmmLibInc.h"
25 #include "External/Common/GmmCachePolicy.h"
26 #include "External/Common/CachePolicy/GmmCachePolicyGen10.h"
27 #include "External/Common/CachePolicy/GmmCachePolicyGen11.h"
28 #include "External/Common/CachePolicy/GmmCachePolicyGen12.h"
29
30 //=============================================================================
31 //
32 // Function: IsSpecialMOCSUsage
33 //
34 // Desc: This function returns special(hw-reserved) MocsIdx based on usage
35 //
36 // Parameters: usage -> Resource usage type
37 // UpdateMOCS -> True if MOCS Table must be updated, ow false
38 //
39 // Return: int32_t
40 //
41 //-----------------------------------------------------------------------------
IsSpecialMOCSUsage(GMM_RESOURCE_USAGE_TYPE Usage,bool & UpdateMOCS)42 int32_t GmmLib::GmmGen12CachePolicy::IsSpecialMOCSUsage(GMM_RESOURCE_USAGE_TYPE Usage, bool &UpdateMOCS)
43 {
44 int32_t MocsIdx = -1;
45 UpdateMOCS = true;
46 //Macros for L3-Eviction Type
47 #define NA 0x0
48 #define RO 0x1
49 #define RW 0x2
50 #define SP 0x3
51
52 switch(Usage)
53 {
54 case GMM_RESOURCE_USAGE_CCS:
55 __GMM_ASSERT(pCachePolicy[Usage].L3 == 0); //Architecturally, CCS isn't L3-cacheable.
56
57 pCachePolicy[Usage].L3 = 0;
58 MocsIdx = 60;
59 break;
60 case GMM_RESOURCE_USAGE_MOCS_62:
61 __GMM_ASSERT(pCachePolicy[Usage].L3 == 0); //Architecturally, TR/Aux-TT node isn't L3-cacheable.
62
63 pCachePolicy[Usage].L3 = 0;
64 MocsIdx = 62;
65 break;
66 case GMM_RESOURCE_USAGE_L3_EVICTION:
67 __GMM_ASSERT(pCachePolicy[Usage].L3 == 0 &&
68 pCachePolicy[Usage].L3Eviction == RW); //Reserved MOCS for L3-evictions
69
70 pCachePolicy[Usage].L3 = 0;
71 pCachePolicy[Usage].L3Eviction = RW;
72 MocsIdx = 63;
73 break;
74 case GMM_RESOURCE_USAGE_L3_EVICTION_SPECIAL:
75 case GMM_RESOURCE_USAGE_CCS_MEDIA_WRITABLE:
76 __GMM_ASSERT(pCachePolicy[Usage].L3 &&
77 pCachePolicy[Usage].L3Eviction == SP); //Reserved MOCS for L3-evictions
78 //Special-case for Displayable, and similar non-LLC accesses
79 GMM_ASSERTDPF(pCachePolicy[Usage].LLC == 0, "MOCS#61's Special Eviction isn't for LLC caching");
80
81 pCachePolicy[Usage].L3 = 1;
82 pCachePolicy[Usage].L3Eviction = SP;
83 MocsIdx = 61;
84 break;
85 default:
86 UpdateMOCS = false;
87 break;
88 }
89
90 if(pCachePolicy[Usage].L3Eviction == RW)
91 {
92 GMM_CACHE_POLICY_ELEMENT L3Eviction;
93 L3Eviction.Value = pCachePolicy[GMM_RESOURCE_USAGE_L3_EVICTION].Value;
94
95 //For internal purpose, hw overrides MOCS#63 as L3-uncacheable, still using it for L3-evictions
96 if(Usage != GMM_RESOURCE_USAGE_L3_EVICTION)
97 {
98 L3Eviction.L3 = 1; //Override L3, to verify MOCS#63 applicable or not
99 }
100
101 __GMM_ASSERT(pCachePolicy[Usage].Value == L3Eviction.Value); //Allow mis-match due to override registries
102 //MocsIdx = 63; //Use non-#63 MOCS, #63 itself is L3-uncached
103 }
104 else if(pCachePolicy[Usage].L3Eviction == SP)
105 {
106 __GMM_ASSERT(pCachePolicy[Usage].Value == pCachePolicy[GMM_RESOURCE_USAGE_L3_EVICTION_SPECIAL].Value); //Allow mis-match due to override registries
107 MocsIdx = 61;
108 }
109
110 return MocsIdx;
111 }
112 //=============================================================================
113 //
114 // Function: __GmmGen12InitCachePolicy
115 //
116 // Desc: This function initializes the cache policy
117 //
118 // Parameters: pCachePolicy -> Ptr to array to be populated with the
119 // mapping of usages -> cache settings.
120 //
121 // Return: GMM_STATUS
122 //
123 //-----------------------------------------------------------------------------
InitCachePolicy()124 GMM_STATUS GmmLib::GmmGen12CachePolicy::InitCachePolicy()
125 {
126
127 __GMM_ASSERTPTR(pCachePolicy, GMM_ERROR);
128
129 #define DEFINE_CACHE_ELEMENT(usage, llc, ellc, l3, wt, age, aom, lecc_scc, l3_scc, scf, sso, cos, hdcl1, l3evict) DEFINE_CP_ELEMENT(usage, llc, ellc, l3, wt, age, aom, lecc_scc, l3_scc, scf, sso, cos, hdcl1, l3evict, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
130
131 #include "GmmGen12CachePolicy.h"
132
133 #define TC_LLC (1)
134 #define TC_ELLC (0)
135 #define TC_LLC_ELLC (2)
136
137 #define LeCC_UNCACHEABLE (0x0)
138 #define LeCC_WC_UNCACHEABLE (0x1)
139 #define LeCC_WT_CACHEABLE (0x2) //Only used as MemPushWRite disqualifier if set along with eLLC-only -still holds on gen12+?
140 #define LeCC_WB_CACHEABLE (0x3)
141
142 #define L3_UNCACHEABLE (0x1)
143 #define L3_WB_CACHEABLE (0x3)
144
145 #define DISABLE_SKIP_CACHING_CONTROL (0x0)
146 #define ENABLE_SKIP_CACHING_CONTROL (0x1)
147
148 #define DISABLE_SELF_SNOOP_OVERRIDE (0x0)
149 #define ENABLE_SELF_SNOOP_OVERRIDE (0x1)
150 #define ENABLE_SELF_SNOOP_ALWAYS (0x3)
151 #define CLASS_SERVICE_ZERO (0x0)
152
153 {
154 SetUpMOCSTable();
155 }
156
157 {
158 // Define index of cache element
159 uint32_t Usage = 0;
160
161 #if(_WIN32 && (_DEBUG || _RELEASE_INTERNAL))
162 void *pKmdGmmContext = NULL;
163
164 OverrideCachePolicy(pKmdGmmContext);
165 #endif
166 // Process the cache policy and fill in the look up table
167 for(; Usage < GMM_RESOURCE_USAGE_MAX; Usage++)
168 {
169 bool CachePolicyError = false;
170 bool SpecialMOCS = false;
171 int32_t CPTblIdx = -1;
172 uint32_t j = 0;
173 uint64_t PTEValue = 0;
174 GMM_CACHE_POLICY_TBL_ELEMENT UsageEle = {0};
175
176 CPTblIdx = IsSpecialMOCSUsage((GMM_RESOURCE_USAGE_TYPE)Usage, SpecialMOCS);
177
178 UsageEle.LeCC.Reserved = 0; // Reserved bits zeroe'd, this is so we
179 // we can compare the unioned LeCC.DwordValue.
180 UsageEle.LeCC.SelfSnoop = DISABLE_SELF_SNOOP_OVERRIDE;
181 UsageEle.LeCC.CoS = CLASS_SERVICE_ZERO;
182 UsageEle.LeCC.SCC = 0;
183 UsageEle.LeCC.ESC = 0;
184
185 if(pCachePolicy[Usage].SCF && pGmmLibContext->GetSkuTable().FtrLLCBypass)
186 {
187 UsageEle.LeCC.SCF = pCachePolicy[Usage].SCF;
188 __GMM_ASSERT(pCachePolicy[Usage].LLC == 0); //LLC and ByPassLLC are mutually-exclusive
189 }
190
191 if(pCachePolicy[Usage].SSO & ENABLE_SELF_SNOOP_OVERRIDE)
192 {
193 UsageEle.LeCC.SelfSnoop = pCachePolicy[Usage].SSO & ENABLE_SELF_SNOOP_ALWAYS;
194 }
195 if(pCachePolicy[Usage].CoS)
196 {
197 UsageEle.LeCC.CoS = pCachePolicy[Usage].CoS;
198 }
199 if(pCachePolicy[Usage].HDCL1)
200 {
201 UsageEle.HDCL1 = 1;
202 }
203 if(pCachePolicy[Usage].LeCC_SCC)
204 {
205 UsageEle.LeCC.SCC = pCachePolicy[Usage].LeCC_SCC;
206 UsageEle.LeCC.ESC = ENABLE_SKIP_CACHING_CONTROL;
207 }
208 UsageEle.LeCC.LRUM = pCachePolicy[Usage].AGE;
209
210 // default to LLC target cache.
211 UsageEle.LeCC.TargetCache = TC_LLC;
212 UsageEle.LeCC.Cacheability = LeCC_WB_CACHEABLE;
213 if(pCachePolicy[Usage].LLC)
214 {
215 UsageEle.LeCC.TargetCache = TC_LLC;
216 __GMM_ASSERT(pCachePolicy[Usage].SCF == 0); //LLC and ByPassLLC are mutually-exclusive
217 }
218 else
219 {
220 UsageEle.LeCC.Cacheability = LeCC_WC_UNCACHEABLE;
221 }
222 UsageEle.L3.Reserved = 0; // Reserved bits zeroe'd, this is so we
223 // we can compare the unioned L3.UshortValue.
224 UsageEle.L3.ESC = DISABLE_SKIP_CACHING_CONTROL;
225 UsageEle.L3.SCC = 0;
226 UsageEle.L3.Cacheability = pCachePolicy[Usage].L3 ? L3_WB_CACHEABLE : L3_UNCACHEABLE;
227
228 __GMM_ASSERT((pCachePolicy[Usage].L3 && pCachePolicy[Usage].L3Eviction != 0) ||
229 (pCachePolicy[Usage].L3 == 0 && (pCachePolicy[Usage].L3Eviction == 0 || Usage == GMM_RESOURCE_USAGE_L3_EVICTION)));
230
231 if(pCachePolicy[Usage].L3_SCC)
232 {
233 UsageEle.L3.ESC = ENABLE_SKIP_CACHING_CONTROL;
234 UsageEle.L3.SCC = (uint16_t)pCachePolicy[Usage].L3_SCC;
235 }
236
237 //Special-case MOCS handling for MOCS Table Index 60-63
238 if(CPTblIdx >= GMM_GEN12_MAX_NUMBER_MOCS_INDEXES)
239 {
240 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &pGmmLibContext->GetCachePolicyTlbElement()[CPTblIdx];
241
242 if(SpecialMOCS &&
243 !(TblEle->LeCC.DwordValue == UsageEle.LeCC.DwordValue &&
244 TblEle->L3.UshortValue == UsageEle.L3.UshortValue &&
245 TblEle->HDCL1 == UsageEle.HDCL1))
246 {
247 //Assert if being overwritten!
248 __GMM_ASSERT(TblEle->LeCC.DwordValue == 0 &&
249 TblEle->L3.UshortValue == 0 &&
250 TblEle->HDCL1 == 0);
251
252 #if(_WIN32 && (_DEBUG || _RELEASE_INTERNAL))
253 if(pCachePolicy[Usage].IsOverridenByRegkey)
254 {
255 TblEle->LeCC.DwordValue = UsageEle.LeCC.DwordValue;
256 TblEle->L3.UshortValue = UsageEle.L3.UshortValue;
257 TblEle->HDCL1 = UsageEle.HDCL1;
258 }
259 #endif
260 }
261 }
262 //For HDC L1 caching, MOCS Table index 48-59 should be used
263 else if(UsageEle.HDCL1)
264 {
265 for(j = GMM_GEN10_HDCL1_MOCS_INDEX_START; j <= CurrentMaxL1HdcMocsIndex; j++)
266 {
267 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &pGmmLibContext->GetCachePolicyTlbElement()[j];
268 if(TblEle->LeCC.DwordValue == UsageEle.LeCC.DwordValue &&
269 TblEle->L3.UshortValue == UsageEle.L3.UshortValue &&
270 TblEle->HDCL1 == UsageEle.HDCL1)
271 {
272 CPTblIdx = j;
273 break;
274 }
275 }
276 }
277 else
278 {
279 // Due to unstable system behavior on TGLLP, MOCS #0 index had to be programmed as UC in MOCS lookup table - pCachePolicyTlbElement
280 // But still Index 0 is Reserved for Error by HW and should not be used.
281 // Hence Gmmlib will opt out from the MOCS#0 usage and Lookup into MOCS table and MOCS index assigment must start from Index 1.
282 for(j = 1; j <= CurrentMaxMocsIndex; j++)
283 {
284 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &pGmmLibContext->GetCachePolicyTlbElement()[j];
285 if(TblEle->LeCC.DwordValue == UsageEle.LeCC.DwordValue &&
286 TblEle->L3.UshortValue == UsageEle.L3.UshortValue &&
287 TblEle->HDCL1 == UsageEle.HDCL1)
288 {
289 CPTblIdx = j;
290 break;
291 }
292 }
293 }
294
295 // Didn't find the caching settings in one of the already programmed lookup table entries.
296 // Need to add a new lookup table entry.
297 if(CPTblIdx == -1)
298 {
299
300 #if(_WIN32 && (_DEBUG || _RELEASE_INTERNAL))
301 // If the Cache Policy setting is overriden through regkey,
302 // don't raise an assert/log error. Raising an assert for debug/perf testing isn't really helpful
303 if(pCachePolicy[Usage].IsOverridenByRegkey)
304 {
305 if(UsageEle.HDCL1 && CurrentMaxL1HdcMocsIndex < GMM_GEN12_MAX_NUMBER_MOCS_INDEXES - 1)
306 {
307 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &(pGmmLibContext->GetCachePolicyTlbElement()[++CurrentMaxL1HdcMocsIndex]);
308 CPTblIdx = CurrentMaxL1HdcMocsIndex;
309
310 TblEle->LeCC.DwordValue = UsageEle.LeCC.DwordValue;
311 TblEle->L3.UshortValue = UsageEle.L3.UshortValue;
312 TblEle->HDCL1 = UsageEle.HDCL1;
313 }
314 else if(CurrentMaxMocsIndex < GMM_GEN10_HDCL1_MOCS_INDEX_START)
315 {
316 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &(pGmmLibContext->GetCachePolicyTlbElement()[++CurrentMaxMocsIndex]);
317 CPTblIdx = CurrentMaxMocsIndex;
318
319 TblEle->LeCC.DwordValue = UsageEle.LeCC.DwordValue;
320 TblEle->L3.UshortValue = UsageEle.L3.UshortValue;
321 TblEle->HDCL1 = UsageEle.HDCL1;
322 }
323 else
324 {
325 // Too many unique caching combinations to program the
326 // MOCS lookup table.
327 CachePolicyError = true;
328 GMM_ASSERTDPF(
329 "Cache Policy Init Error: Invalid Cache Programming, too many unique caching combinations"
330 "(we only support GMM_GEN_MAX_NUMBER_MOCS_INDEXES = %d)",
331 GMM_MAX_NUMBER_MOCS_INDEXES - 1);
332 // Set cache policy index to uncached.
333 CPTblIdx = 3;
334 }
335 }
336 else
337 #endif
338 {
339 GMM_ASSERTDPF(false, "CRITICAL ERROR: Cache Policy Usage value specified by Client is not defined in Fixed MOCS Table!");
340
341 CachePolicyError = true;
342 GMM_ASSERTDPF(
343 "Cache Policy Init Error: Invalid Cache Programming, too many unique caching combinations"
344 "(we only support GMM_GEN_MAX_NUMBER_MOCS_INDEXES = %d)",
345 CurrentMaxMocsIndex);
346
347 // Set cache policy index to uncached.
348 CPTblIdx = 3;
349 }
350 }
351
352 // PTE entries do not control caching on SKL+ (for legacy context)
353 if(!GetUsagePTEValue(pCachePolicy[Usage], Usage, &PTEValue))
354 {
355 CachePolicyError = true;
356 }
357
358 pCachePolicy[Usage].PTE.DwordValue = PTEValue & 0xFFFFFFFF;
359 pCachePolicy[Usage].PTE.HighDwordValue = 0;
360
361 pCachePolicy[Usage].MemoryObjectOverride.Gen12.Index = CPTblIdx;
362
363 pCachePolicy[Usage].Override = ALWAYS_OVERRIDE;
364
365 if(CachePolicyError)
366 {
367 GMM_ASSERTDPF("Cache Policy Init Error: Invalid Cache Programming - Element %d", Usage);
368 }
369 }
370 }
371
372 return GMM_SUCCESS;
373 }
374
375 /////////////////////////////////////////////////////////////////////////////////////
376 /// Return true if (MT2) is a better match for (WantedMT)
377 /// than (MT1)
378 ///
379 /// @param[in] WantedMT: Wanted Memory Type
380 /// @param[in] MT1: Memory Type for PATIdx1
381 /// @param[in] MT2: Memory Type for PATIdx2
382 ///
383 /// @return Select the new PAT Index True/False
384 /////////////////////////////////////////////////////////////////////////////////////
SelectNewPATIdx(GMM_GFX_MEMORY_TYPE WantedMT,GMM_GFX_MEMORY_TYPE MT1,GMM_GFX_MEMORY_TYPE MT2)385 uint8_t GmmLib::GmmGen12CachePolicy::SelectNewPATIdx(GMM_GFX_MEMORY_TYPE WantedMT,
386 GMM_GFX_MEMORY_TYPE MT1, GMM_GFX_MEMORY_TYPE MT2)
387 {
388 uint8_t SelectPAT2 = 0;
389
390 // select on Memory Type
391 if(MT1 != WantedMT)
392 {
393 if(MT2 == WantedMT || MT2 == GMM_GFX_UC_WITH_FENCE)
394 {
395 SelectPAT2 = 1;
396 }
397 goto EXIT;
398 }
399
400 EXIT:
401 return SelectPAT2;
402 }
403
404 /////////////////////////////////////////////////////////////////////////////////////
405 /// Returns the PAT idx that best matches the cache policy for this usage.
406 ///
407 /// @param: CachePolicy: cache policy for a usage
408 ///
409 /// @return PAT Idx to use in the PTE
410 /////////////////////////////////////////////////////////////////////////////////////
BestMatchingPATIdx(GMM_CACHE_POLICY_ELEMENT CachePolicy)411 uint32_t GmmLib::GmmGen12CachePolicy::BestMatchingPATIdx(GMM_CACHE_POLICY_ELEMENT CachePolicy)
412 {
413 uint32_t i;
414 uint32_t PATIdx = 0;
415 GMM_GFX_MEMORY_TYPE WantedMemoryType = GMM_GFX_UC_WITH_FENCE, MemoryType;
416 WA_TABLE * pWaTable = &const_cast<WA_TABLE &>(pGmmLibContext->GetWaTable());
417
418 WantedMemoryType = GetWantedMemoryType(CachePolicy);
419
420 // Override wantedMemoryType so that PAT.MT is UC
421 // Gen12 uses max function to resolve PAT-vs-MOCS MemType, So unless PTE.PAT says UC, MOCS won't be able to set UC!
422 if(pWaTable->WaMemTypeIsMaxOfPatAndMocs)
423 {
424 WantedMemoryType = GMM_GFX_UC_WITH_FENCE;
425 }
426
427 for(i = 1; i < NumPATRegisters; i++)
428 {
429 GMM_PRIVATE_PAT PAT1 = GetPrivatePATEntry(PATIdx);
430 GMM_PRIVATE_PAT PAT2 = GetPrivatePATEntry(i);
431
432 if(SelectNewPATIdx(WantedMemoryType,
433 (GMM_GFX_MEMORY_TYPE)PAT1.Gen12.MemoryType,
434 (GMM_GFX_MEMORY_TYPE)PAT2.Gen12.MemoryType))
435 {
436 PATIdx = i;
437 }
438 }
439
440 MemoryType = (GMM_GFX_MEMORY_TYPE)GetPrivatePATEntry(PATIdx).Gen12.MemoryType;
441
442 if(MemoryType != WantedMemoryType)
443 {
444 // Failed to find a matching PAT entry
445 return GMM_PAT_ERROR;
446 }
447 return PATIdx;
448 }
449
450 /////////////////////////////////////////////////////////////////////////////////////
451 /// Initializes WA's needed for setting up the Private PATs
452 /// WaNoMocsEllcOnly (reset)
453 /// WaGttPat0, WaGttPat0GttWbOverOsIommuEllcOnly, WaGttPat0WB (use from base class)
454 ///
455 /// @return GMM_STATUS
456 ///
457 /////////////////////////////////////////////////////////////////////////////////////
SetPATInitWA()458 GMM_STATUS GmmLib::GmmGen12CachePolicy::SetPATInitWA()
459 {
460 GMM_STATUS Status = GMM_SUCCESS;
461 WA_TABLE * pWaTable = &const_cast<WA_TABLE &>(pGmmLibContext->GetWaTable());
462
463 #if(defined(__GMM_KMD__))
464 __GMM_ASSERT(pGmmLibContext->GetSkuTable().FtrMemTypeMocsDeferPAT == 0x0); //MOCS.TargetCache supports eLLC only, PAT.TC -> reserved bits.
465 pWaTable->WaGttPat0WB = 0; //Override PAT #0
466 #else
467 Status = GMM_ERROR;
468 #endif
469
470 return Status;
471 }
472
473 /////////////////////////////////////////////////////////////////////////////////////
474 /// Initializes the Gfx PAT tables for AdvCtx and Gfx MMIO/Private PAT
475 /// PAT0 = WB_COHERENT or UC depending on WaGttPat0WB
476 /// PAT1 = UC or WB_COHERENT depending on WaGttPat0WB
477 /// PAT2 = WB_MOCSLESS
478 /// PAT3 = WB
479 /// PAT4 = WT
480 /// PAT5 = WC
481 /// PAT6 = WC
482 /// PAT7 = WC
483 /// HLD says to set to PAT0/1 to WC, but since we don't have a WC in GPU,
484 /// WC option is same as UC. Hence setting PAT0 or PAT1 to UC.
485 /// Unused PAT's (5,6,7) are set to WC.
486 ///
487 /// @return GMM_STATUS
488 /////////////////////////////////////////////////////////////////////////////////////
SetupPAT()489 GMM_STATUS GmmLib::GmmGen12CachePolicy::SetupPAT()
490 {
491 GMM_STATUS Status = GMM_SUCCESS;
492 #if(defined(__GMM_KMD__))
493 uint32_t i = 0;
494
495 GMM_GFX_MEMORY_TYPE GfxMemType = GMM_GFX_UC_WITH_FENCE;
496 int32_t * pPrivatePATTableMemoryType = NULL;
497 pPrivatePATTableMemoryType = pGmmLibContext->GetPrivatePATTableMemoryType();
498
499 __GMM_ASSERT(pGmmLibContext->GetSkuTable().FtrIA32eGfxPTEs);
500
501 for(i = 0; i < GMM_NUM_GFX_PAT_TYPES; i++)
502 {
503 pPrivatePATTableMemoryType[i] = -1;
504 }
505
506 // Set values for GmmGlobalInfo PrivatePATTable
507 for(i = 0; i < NumPATRegisters; i++)
508 {
509 GMM_PRIVATE_PAT PAT = {0};
510
511 switch(i)
512 {
513 case PAT0:
514 if(pGmmLibContext->GetWaTable().WaGttPat0)
515 {
516 if(pGmmLibContext->GetWaTable().WaGttPat0WB)
517 {
518 GfxMemType = GMM_GFX_WB;
519 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
520 }
521 else
522 {
523 GfxMemType = GMM_GFX_UC_WITH_FENCE;
524 pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT0;
525 }
526 }
527 else // if GTT is not tied to PAT0 then WaGttPat0WB is NA
528 {
529 GfxMemType = GMM_GFX_WB;
530 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
531 }
532 break;
533
534 case PAT1:
535 if(pGmmLibContext->GetWaTable().WaGttPat0 && !pGmmLibContext->GetWaTable().WaGttPat0WB)
536 {
537 GfxMemType = GMM_GFX_WB;
538 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT1;
539 }
540 else
541 {
542 GfxMemType = GMM_GFX_UC_WITH_FENCE;
543 pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT1;
544 }
545 break;
546
547 case PAT2:
548 // This PAT idx shall be used for MOCS'Less resources like Page Tables
549 // Page Tables have TC hardcoded to eLLC+LLC in Adv Ctxt. Hence making this to have same in Leg Ctxt.
550 // For BDW-H, due to Perf issue, TC has to be eLLC only for Page Tables when eDRAM is present.
551 GfxMemType = GMM_GFX_WB;
552 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_MOCSLESS] = PAT2;
553 break;
554
555 case PAT3:
556 GfxMemType = GMM_GFX_WB;
557 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB] = PAT3;
558 break;
559
560 case PAT4:
561 GfxMemType = GMM_GFX_WT;
562 pPrivatePATTableMemoryType[GMM_GFX_PAT_WT] = PAT4;
563 break;
564
565 case PAT5:
566 case PAT6:
567 case PAT7:
568 GfxMemType = GMM_GFX_WC;
569 pPrivatePATTableMemoryType[GMM_GFX_PAT_WC] = PAT5;
570 break;
571
572 default:
573 __GMM_ASSERT(0);
574 Status = GMM_ERROR;
575 }
576
577 PAT.Gen12.MemoryType = GfxMemType;
578
579 SetPrivatePATEntry(i, PAT);
580 }
581
582 #else
583 Status = GMM_ERROR;
584 #endif
585 return Status;
586 }
587
588 //=============================================================================
589 //
590 // Function: SetUpMOCSTable
591 //
592 // Desc:
593 //
594 // Parameters:
595 //
596 // Return: GMM_STATUS
597 //
598 //-----------------------------------------------------------------------------
SetUpMOCSTable()599 void GmmLib::GmmGen12CachePolicy::SetUpMOCSTable()
600 {
601 GMM_CACHE_POLICY_TBL_ELEMENT *pCachePolicyTlbElement = &(pGmmLibContext->GetCachePolicyTlbElement()[0]);
602
603 #define GMM_DEFINE_MOCS(Index, L3_ESC, L3_SCC, L3_CC, LeCC_CC, LeCC_TC, LeCC_LRUM, LeCC_AOM, LeCC_ESC, LeCC_SCC, LeCC_PFM, LeCC_SCF, LeCC_CoS, LeCC_SelfSnoop, _HDCL1) \
604 { \
605 pCachePolicyTlbElement[Index].L3.ESC = L3_ESC; \
606 pCachePolicyTlbElement[Index].L3.SCC = L3_SCC; \
607 pCachePolicyTlbElement[Index].L3.Cacheability = L3_CC; \
608 pCachePolicyTlbElement[Index].LeCC.Cacheability = LeCC_CC; \
609 pCachePolicyTlbElement[Index].LeCC.TargetCache = LeCC_TC; \
610 pCachePolicyTlbElement[Index].LeCC.LRUM = LeCC_LRUM; \
611 pCachePolicyTlbElement[Index].LeCC.AOM = LeCC_AOM; \
612 pCachePolicyTlbElement[Index].LeCC.ESC = LeCC_ESC; \
613 pCachePolicyTlbElement[Index].LeCC.SCC = LeCC_SCC; \
614 pCachePolicyTlbElement[Index].LeCC.PFM = LeCC_PFM; \
615 pCachePolicyTlbElement[Index].LeCC.SCF = LeCC_SCF; \
616 pCachePolicyTlbElement[Index].LeCC.CoS = LeCC_CoS; \
617 pCachePolicyTlbElement[Index].LeCC.SelfSnoop = LeCC_SelfSnoop; \
618 pCachePolicyTlbElement[Index].HDCL1 = _HDCL1; \
619 }
620
621 // clang-format off
622
623 //Default MOCS Table
624 for(int index = 0; index < GMM_MAX_NUMBER_MOCS_INDEXES; index++)
625 { // Index ES SCC L3CC LeCC TC LRUM DAoM ERSC SCC PFM SCF CoS SSE HDCL1
626 GMM_DEFINE_MOCS( index , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
627 }
628
629 // Fixed MOCS Table
630 // Index ESC SCC L3CC LeCC TC LRUM DAoM ERSC SCC PFM SCF CoS SSE HDCL1
631 GMM_DEFINE_MOCS( 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
632 GMM_DEFINE_MOCS( 2 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
633 GMM_DEFINE_MOCS( 3 , 0 , 0 , 1 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
634 GMM_DEFINE_MOCS( 4 , 0 , 0 , 3 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
635 GMM_DEFINE_MOCS( 5 , 0 , 0 , 1 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
636 GMM_DEFINE_MOCS( 6 , 0 , 0 , 1 , 3 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
637 GMM_DEFINE_MOCS( 7 , 0 , 0 , 3 , 3 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
638 GMM_DEFINE_MOCS( 8 , 0 , 0 , 1 , 3 , 1 , 2 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
639 GMM_DEFINE_MOCS( 9 , 0 , 0 , 3 , 3 , 1 , 2 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
640 GMM_DEFINE_MOCS( 10 , 0 , 0 , 1 , 3 , 1 , 3 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
641 GMM_DEFINE_MOCS( 11 , 0 , 0 , 3 , 3 , 1 , 3 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
642 GMM_DEFINE_MOCS( 12 , 0 , 0 , 1 , 3 , 1 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
643 GMM_DEFINE_MOCS( 13 , 0 , 0 , 3 , 3 , 1 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
644 GMM_DEFINE_MOCS( 14 , 0 , 0 , 1 , 3 , 1 , 2 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
645 GMM_DEFINE_MOCS( 15 , 0 , 0 , 3 , 3 , 1 , 2 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
646 GMM_DEFINE_MOCS( 16 , 0 , 0 , 1 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 1 , 0 , 0 , 0 )
647 GMM_DEFINE_MOCS( 17 , 0 , 0 , 3 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 1 , 0 , 0 , 0 )
648 GMM_DEFINE_MOCS( 18 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 3 , 0 )
649 GMM_DEFINE_MOCS( 19 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 0 , 7 , 0 , 0 , 0 , 0 , 0 )
650 GMM_DEFINE_MOCS( 20 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 0 , 3 , 0 , 0 , 0 , 0 , 0 )
651 GMM_DEFINE_MOCS( 21 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 0 , 1 , 0 , 0 , 0 , 0 , 0 )
652 GMM_DEFINE_MOCS( 22 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 1 , 3 , 0 , 0 , 0 , 0 , 0 )
653 GMM_DEFINE_MOCS( 23 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 1 , 7 , 0 , 0 , 0 , 0 , 0 )
654 GMM_DEFINE_MOCS( 48 , 0 , 0 , 3 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 1 )
655 GMM_DEFINE_MOCS( 49 , 0 , 0 , 3 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 1 )
656 GMM_DEFINE_MOCS( 50 , 0 , 0 , 1 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 1 )
657 GMM_DEFINE_MOCS( 51 , 0 , 0 , 1 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 1 )
658 GMM_DEFINE_MOCS( 60 , 0 , 0 , 1 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
659 GMM_DEFINE_MOCS( 61 , 0 , 0 , 3 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 1 , 0 , 0 , 0 )
660 GMM_DEFINE_MOCS( 62 , 0 , 0 , 1 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
661 GMM_DEFINE_MOCS( 63 , 0 , 0 , 1 , 3 , 1 , 3 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
662
663
664 if(!pGmmLibContext->GetSkuTable().FtrLLCBypass ||
665 GFX_GET_CURRENT_PRODUCT(pGmmLibContext->GetPlatformInfo().Platform) == IGFX_ROCKETLAKE)
666 {
667 GMM_DEFINE_MOCS( 16 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
668 GMM_DEFINE_MOCS( 17 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
669 GMM_DEFINE_MOCS( 61 , 0 , 0 , 3 , 1 , 1 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 )
670 }
671 // clang-format on
672
673 CurrentMaxMocsIndex = 23;
674 CurrentMaxL1HdcMocsIndex = 51;
675 CurrentMaxSpecialMocsIndex = 63;
676
677 #undef GMM_DEFINE_MOCS
678 }
679