1 /*==============================================================================
2 Copyright(c) 2017 Intel Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a
5 copy of this software and associated documentation files(the "Software"),
6 to deal in the Software without restriction, including without limitation
7 the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and / or sell copies of the Software, and to permit persons to whom the
9 Software is furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included
12 in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 OTHER DEALINGS IN THE SOFTWARE.
21 ============================================================================*/
22
23 #include "Internal/Common/GmmLibInc.h"
24 #include "External/Common/GmmCachePolicy.h"
25
26 //=============================================================================
27 //
28 // Function: __GmmGen8InitCachePolicy
29 //
30 // Desc: This function initializes the cache policy
31 //
32 // Parameters: pCachePolicy -> Ptr to array to be populated with the
33 // mapping of usages -> cache settings.
34 //
35 // Return: GMM_STATUS
36 //-----------------------------------------------------------------------------
InitCachePolicy()37 GMM_STATUS GmmLib::GmmGen8CachePolicy::InitCachePolicy()
38 {
39
40 __GMM_ASSERTPTR(pCachePolicy, GMM_ERROR);
41
42 #define DEFINE_CACHE_ELEMENT(usage, llc, ellc, l3, wt, age) DEFINE_CP_ELEMENT(usage, llc, ellc, l3, wt, age, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
43 #include "GmmGen8CachePolicy.h"
44
45 {
46 // Gen8 Memory Object Definitions
47 #define MO_ELLC 0x0
48 #define MO_LLC 0x1
49 #define MO_LLC_ELLC 0x2
50 #define MO_L3_LLC_ELLC 0x3
51
52 #define MO_USE_PTE 0x0
53 #define MO_UC 0x1
54 #define MO_WT 0x2
55 #define MO_WB 0x3
56
57 // Define index of cache element
58 uint32_t Usage = 0;
59
60 // Process Cache Policy and fill in look up table
61 for(; Usage < GMM_RESOURCE_USAGE_MAX; Usage++)
62 {
63 bool CachePolicyError = false;
64 uint64_t PTEValue = 0;
65
66 if(pCachePolicy[Usage].LLC && pCachePolicy[Usage].ELLC && pCachePolicy[Usage].L3)
67 pCachePolicy[Usage].MemoryObjectOverride.Gen8.TargetCache = MO_L3_LLC_ELLC;
68 else if(pCachePolicy[Usage].LLC && pCachePolicy[Usage].ELLC)
69 pCachePolicy[Usage].MemoryObjectOverride.Gen8.TargetCache = MO_LLC_ELLC;
70 else if(pCachePolicy[Usage].ELLC)
71 pCachePolicy[Usage].MemoryObjectOverride.Gen8.TargetCache = MO_ELLC;
72 else if(pCachePolicy[Usage].LLC)
73 pCachePolicy[Usage].MemoryObjectOverride.Gen8.TargetCache = MO_LLC;
74
75 pCachePolicy[Usage].MemoryObjectOverride.Gen8.Age = pCachePolicy[Usage].AGE;
76
77 if(pCachePolicy[Usage].WT)
78 pCachePolicy[Usage].MemoryObjectOverride.Gen8.CacheControl = MO_WT;
79 // L3 is not included because WT vs UC vs WB only effects uncore
80 else if(!(pCachePolicy[Usage].LLC || pCachePolicy[Usage].ELLC))
81 pCachePolicy[Usage].MemoryObjectOverride.Gen8.CacheControl = MO_UC;
82
83 else
84 pCachePolicy[Usage].MemoryObjectOverride.Gen8.CacheControl = MO_WB;
85
86
87 if(!GetUsagePTEValue(pCachePolicy[Usage], Usage, &PTEValue))
88 {
89 CachePolicyError = true;
90 }
91 // On error, the PTE value is set to a UC PAT entry
92 pCachePolicy[Usage].PTE.DwordValue = PTEValue & 0xFFFFFFFF;
93 pCachePolicy[Usage].PTE.HighDwordValue = 0;
94 pCachePolicy[Usage].Override = ALWAYS_OVERRIDE;
95
96 if(CachePolicyError)
97 {
98 GMM_ASSERTDPF("Cache Policy Init Error: Invalid Cache Programming - Element %d", Usage);
99 }
100 }
101 }
102
103 return GMM_SUCCESS;
104 }
105
106 /////////////////////////////////////////////////////////////////////////////////////
107 /// Returns true if usage PTE entries are set for caching, false otherwise.
108 ///
109 /// @param[in] Usage: type of usage
110 ///
111 /// @return true if the usage PTE entry is set for cached, false otherwise.
112 /////////////////////////////////////////////////////////////////////////////////////
CachePolicyIsUsagePTECached(GMM_RESOURCE_USAGE_TYPE Usage)113 uint8_t GMM_STDCALL GmmLib::GmmGen8CachePolicy::CachePolicyIsUsagePTECached(GMM_RESOURCE_USAGE_TYPE Usage)
114 {
115 GMM_UNREFERENCED_PARAMETER(Usage);
116 return 0;
117 }
118
119 /////////////////////////////////////////////////////////////////////////////////////
120 /// Initializes the Gfx PAT tables for AdvCtx and Gfx MMIO/Private PAT
121 /// PAT0 = WB_COHERENT or UC depending on WaGttPat0WB
122 /// PAT1 = UC or WB_COHERENT depending on WaGttPat0WB
123 /// PAT2 = WB_MOCSLESS, with TC = eLLC+LLC
124 /// PAT3 = WB
125 /// PAT4 = WT
126 /// PAT5 = WC
127 /// PAT6 = WC
128 /// PAT7 = WC
129 /// HLD says to set to PAT0/1 to WC, but since we don't have a WC in GPU,
130 /// WC option is same as UC. Hence setting PAT0 or PAT1 to UC.
131 /// Unused PAT's (5,6,7) are set to WC.
132 ///
133 /// @return GMM_STATUS
134 /////////////////////////////////////////////////////////////////////////////////////
SetupPAT()135 GMM_STATUS GmmLib::GmmGen8CachePolicy::SetupPAT()
136 {
137 GMM_STATUS Status = GMM_SUCCESS;
138 #if(defined(__GMM_KMD__))
139 uint32_t i = 0;
140
141 GMM_GFX_MEMORY_TYPE GfxMemType = GMM_GFX_UC_WITH_FENCE;
142 // No optional selection on Age or Target Cache because for an SVM-OS Age and
143 // Target Cache would not work [for an SVM-OS the Page Table is shared with IA
144 // and we don't have control of the PAT Idx]. If there is a strong ask from D3D
145 // or the performance analysis team, Age could be added.
146 // Add Class of Service when required.
147 GMM_GFX_TARGET_CACHE GfxTargetCache = GMM_GFX_TC_ELLC_LLC;
148 uint8_t Age = 1;
149 uint8_t ServiceClass = 0;
150 int32_t * pPrivatePATTableMemoryType = NULL;
151
152 pPrivatePATTableMemoryType = pGmmLibContext->GetPrivatePATTableMemoryType();
153
154 __GMM_ASSERT(pGmmLibContext->GetSkuTable().FtrIA32eGfxPTEs);
155
156 for(i = 0; i < GMM_NUM_GFX_PAT_TYPES; i++)
157 {
158 pPrivatePATTableMemoryType[i] = -1;
159 }
160
161 // Set values for GmmGlobalInfo PrivatePATTable
162 for(i = 0; i < NumPATRegisters; i++)
163 {
164 GMM_PRIVATE_PAT PAT = {0};
165
166 if(pGmmLibContext->GetWaTable().WaNoMocsEllcOnly)
167 {
168 GfxTargetCache = GMM_GFX_TC_ELLC_ONLY;
169 }
170 else
171 {
172 GfxTargetCache = GMM_GFX_TC_ELLC_LLC;
173 }
174
175 switch(i)
176 {
177 case PAT0:
178 if(pGmmLibContext->GetWaTable().WaGttPat0)
179 {
180 if(pGmmLibContext->GetWaTable().WaGttPat0WB)
181 {
182 GfxMemType = GMM_GFX_WB;
183 if(GFX_IS_ATOM_PLATFORM(pGmmLibContext))
184 {
185 PAT.PreGen10.Snoop = 1;
186 }
187 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
188 }
189 else
190 {
191 GfxMemType = GMM_GFX_UC_WITH_FENCE;
192 pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT0;
193 }
194 }
195 else // if GTT is not tied to PAT0 then WaGttPat0WB is NA
196 {
197 GfxMemType = GMM_GFX_WB;
198 if(GFX_IS_ATOM_PLATFORM(pGmmLibContext))
199 {
200 PAT.PreGen10.Snoop = 1;
201 }
202 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
203 }
204 break;
205
206 case PAT1:
207 if(pGmmLibContext->GetWaTable().WaGttPat0 && !pGmmLibContext->GetWaTable().WaGttPat0WB)
208 {
209 GfxMemType = GMM_GFX_WB;
210 if(GFX_IS_ATOM_PLATFORM(pGmmLibContext))
211 {
212 PAT.PreGen10.Snoop = 1;
213 }
214 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT1;
215 }
216 else
217 {
218 GfxMemType = GMM_GFX_UC_WITH_FENCE;
219 pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT1;
220 }
221 break;
222
223 case PAT2:
224 // This PAT idx shall be used for MOCS'Less resources like Page Tables
225 // Page Tables have TC hardcoded to eLLC+LLC in Adv Ctxt. Hence making this to have same in Leg Ctxt.
226 // For BDW-H, due to Perf issue, TC has to be eLLC only for Page Tables when eDRAM is present.
227 GfxMemType = GMM_GFX_WB;
228
229 if(pGmmLibContext->GetWaTable().WaNoMocsEllcOnly)
230 {
231 GfxTargetCache = GMM_GFX_TC_ELLC_ONLY;
232 }
233 else
234 {
235 GfxTargetCache = GMM_GFX_TC_ELLC_LLC;
236 }
237
238 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_MOCSLESS] = PAT2;
239 break;
240
241 case PAT3:
242 GfxMemType = GMM_GFX_WB;
243 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB] = PAT3;
244 break;
245
246 case PAT4:
247 GfxMemType = GMM_GFX_WT;
248 pPrivatePATTableMemoryType[GMM_GFX_PAT_WT] = PAT4;
249 break;
250
251 case PAT5:
252 case PAT6:
253 case PAT7:
254 GfxMemType = GMM_GFX_WC;
255 pPrivatePATTableMemoryType[GMM_GFX_PAT_WC] = PAT5;
256 break;
257
258 default:
259 __GMM_ASSERT(0);
260 Status = GMM_ERROR;
261 }
262
263 PAT.PreGen10.MemoryType = GfxMemType;
264 PAT.PreGen10.TargetCache = GfxTargetCache;
265 PAT.PreGen10.Age = Age;
266
267 SetPrivatePATEntry(i, PAT);
268 }
269
270 #else
271 Status = GMM_ERROR;
272 #endif
273 return Status;
274 }
275
276 /////////////////////////////////////////////////////////////////////////////////////
277 /// Initializes WA's needed for setting up the Private PATs
278 /// WaNoMocsEllcOnly, WaGttPat0, WaGttPat0GttWbOverOsIommuEllcOnly, WaGttPat0WB
279 ///
280 /// @return GMM_STATUS
281 ///
282 /////////////////////////////////////////////////////////////////////////////////////
SetPATInitWA()283 GMM_STATUS GmmLib::GmmGen8CachePolicy::SetPATInitWA()
284 {
285 GMM_STATUS Status = GMM_SUCCESS;
286 WA_TABLE * pWaTable = &const_cast<WA_TABLE &>(pGmmLibContext->GetWaTable());
287
288 #if(defined(__GMM_KMD__))
289
290 pWaTable->WaGttPat0 = 1;
291 pWaTable->WaGttPat0WB = 1;
292 pWaTable->WaGttPat0GttWbOverOsIommuEllcOnly = 1;
293
294 // Platforms which support OS-IOMMU.
295 if(pGmmLibContext->GetSkuTable().FtrWddm2Svm)
296 {
297 pWaTable->WaGttPat0GttWbOverOsIommuEllcOnly = 0;
298 pWaTable->WaGttPat0WB = 0;
299 }
300
301 #else
302 Status = GMM_ERROR;
303 #endif
304
305 return Status;
306 }
307
308 /////////////////////////////////////////////////////////////////////////////////////
309 /// Returns the PAT idx that best matches the cache policy for this usage.
310 ///
311 /// @param: CachePolicy: cache policy for a usage
312 ///
313 /// @return PAT Idx to use in the PTE
314 /////////////////////////////////////////////////////////////////////////////////////
BestMatchingPATIdx(GMM_CACHE_POLICY_ELEMENT CachePolicy)315 uint32_t GmmLib::GmmGen8CachePolicy::BestMatchingPATIdx(GMM_CACHE_POLICY_ELEMENT CachePolicy)
316 {
317 uint32_t i;
318 uint32_t PATIdx = 0;
319 GMM_GFX_MEMORY_TYPE WantedMemoryType = GMM_GFX_UC_WITH_FENCE, MemoryType;
320 GMM_GFX_TARGET_CACHE WantedTC = GMM_GFX_TC_ELLC_LLC;
321
322 WantedMemoryType = GetWantedMemoryType(CachePolicy);
323
324 if(CachePolicy.LLC && CachePolicy.ELLC)
325 {
326 WantedTC = GMM_GFX_TC_ELLC_LLC;
327 }
328 else if(CachePolicy.LLC)
329 {
330 WantedTC = GMM_GFX_TC_LLC_ONLY;
331 }
332 else if(CachePolicy.ELLC)
333 {
334 WantedTC = GMM_GFX_TC_ELLC_ONLY; // Note: this overrides the MOCS target cache selection.
335 }
336
337 for(i = 1; i < NumPATRegisters; i++)
338 {
339 GMM_PRIVATE_PAT PAT1 = GetPrivatePATEntry(PATIdx);
340 GMM_PRIVATE_PAT PAT2 = GetPrivatePATEntry(i);
341
342 if(SelectNewPATIdx(WantedMemoryType, WantedTC,
343 (GMM_GFX_MEMORY_TYPE)PAT1.PreGen10.MemoryType, (GMM_GFX_TARGET_CACHE)PAT1.PreGen10.TargetCache,
344 (GMM_GFX_MEMORY_TYPE)PAT2.PreGen10.MemoryType, (GMM_GFX_TARGET_CACHE)PAT2.PreGen10.TargetCache))
345 {
346 PATIdx = i;
347 }
348 }
349
350 MemoryType = (GMM_GFX_MEMORY_TYPE)GetPrivatePATEntry(PATIdx).PreGen10.MemoryType;
351
352 if(MemoryType != WantedMemoryType)
353 {
354 // Failed to find a matching PAT entry
355 return GMM_PAT_ERROR;
356 }
357 return PATIdx;
358 }
359
360 /////////////////////////////////////////////////////////////////////////////////////
361 /// Sets the GMM Private PAT in the PrivatePATTable for the PATIdx, GMM_PRIVATE_PAT
362 /// Entry passed
363 ///
364 /// @param[in] PATIdx
365 /// @param[in] GMM_PRIVATE_PAT: PAT Entry
366 ///
367 /// @return Pass/ fail
368 /////////////////////////////////////////////////////////////////////////////////////
SetPrivatePATEntry(uint32_t PATIdx,GMM_PRIVATE_PAT Entry)369 bool GmmLib::GmmGen8CachePolicy::SetPrivatePATEntry(uint32_t PATIdx, GMM_PRIVATE_PAT Entry)
370 {
371 if(PATIdx >= NumPATRegisters)
372 {
373 GMM_ASSERTDPF(false, "CRITICAL ERROR: INVALID PAT IDX");
374 return false;
375 }
376
377 pGmmLibContext->GetPrivatePATTable()[PATIdx] = Entry;
378 return true;
379 }
380
381 /////////////////////////////////////////////////////////////////////////////////////
382 /// Gets the GMM Private PAT from the PrivatePATTable for the PATIdx passed
383 ///
384 /// @param[in] PATIdx
385 ///
386 /// @return GMM_PRIVATE_PAT: Entry
387 /////////////////////////////////////////////////////////////////////////////////////
GetPrivatePATEntry(uint32_t PATIdx)388 GMM_PRIVATE_PAT GmmLib::GmmGen8CachePolicy::GetPrivatePATEntry(uint32_t PATIdx)
389 {
390 GMM_PRIVATE_PAT NullPAT = {0};
391
392 if(PATIdx >= NumPATRegisters)
393 {
394 GMM_ASSERTDPF(false, "CRITICAL ERROR: INVALID PAT IDX");
395 return NullPAT;
396 }
397 return pGmmLibContext->GetPrivatePATTable()[PATIdx];
398 }
399
400 /////////////////////////////////////////////////////////////////////////////////////
401 /// Return true if (MT2, TC2) is a better match for (WantedMT, WantedTC)
402 /// than (MT1, TC1)
403 ///
404 /// @param[in] WantedMT: Wanted Memory Type
405 /// @param[in] WantedTC: Wanted Target Cache
406 /// @param[in] MT1: Memory Type for PATIdx1
407 /// @param[in] TC1: Target Cache for PATIdx1
408 /// @param[in] MT2: Memory Type for PATIdx2
409 /// @param[in] TC2: Target Cache for PATIdx2
410 ///
411 /// @return Select the new PAT Index True/False
412 /////////////////////////////////////////////////////////////////////////////////////
SelectNewPATIdx(GMM_GFX_MEMORY_TYPE WantedMT,GMM_GFX_TARGET_CACHE WantedTC,GMM_GFX_MEMORY_TYPE MT1,GMM_GFX_TARGET_CACHE TC1,GMM_GFX_MEMORY_TYPE MT2,GMM_GFX_TARGET_CACHE TC2)413 bool GmmLib::GmmGen8CachePolicy::SelectNewPATIdx(GMM_GFX_MEMORY_TYPE WantedMT, GMM_GFX_TARGET_CACHE WantedTC,
414 GMM_GFX_MEMORY_TYPE MT1, GMM_GFX_TARGET_CACHE TC1,
415 GMM_GFX_MEMORY_TYPE MT2, GMM_GFX_TARGET_CACHE TC2)
416 {
417 bool SelectPAT2 = false;
418
419 // select on Memory Type
420 if(MT1 != WantedMT)
421 {
422 if(MT2 == WantedMT || MT2 == GMM_GFX_UC_WITH_FENCE)
423 {
424 SelectPAT2 = true;
425 }
426 goto EXIT;
427 }
428
429 // select on Target Cache
430 if(WantedTC != TC1)
431 {
432 if(WantedMT == MT2 && WantedTC == TC2)
433 {
434 SelectPAT2 = true;
435 }
436 goto EXIT;
437 }
438
439 EXIT:
440 return SelectPAT2;
441 }
442
443 /////////////////////////////////////////////////////////////////////////////////////
444 /// Returns PTE value
445 ///
446 /// @param[in] CachePolicyUsage: Cache Policy for Usage
447 ///
448 /// @return true: success, false: failure
449 /////////////////////////////////////////////////////////////////////////////////////
GetUsagePTEValue(GMM_CACHE_POLICY_ELEMENT CachePolicyUsage,uint32_t Usage,uint64_t * pPTEDwordValue)450 bool GmmLib::GmmGen8CachePolicy::GetUsagePTEValue(GMM_CACHE_POLICY_ELEMENT CachePolicyUsage,
451 uint32_t Usage,
452 uint64_t * pPTEDwordValue)
453 {
454 GMM_PTE_CACHE_CONTROL_BITS PTE = {0};
455 bool Success = true;
456 uint32_t PATIdx = 0;
457
458 // Don't setup PTE values in UMD
459 #if __GMM_KMD__
460 if((PATIdx = BestMatchingPATIdx(CachePolicyUsage)) == GMM_PAT_ERROR)
461 {
462 // IAe32 PAT table does not necessarily have an entry for WT memory type
463 // => not a cache policy init error if WT is unavailable.
464 Success = CachePolicyUsage.WT ? true : false;
465
466 // degrade to UC
467 {
468 GMM_CACHE_POLICY_ELEMENT CachePolicyElement = {0};
469
470 const char *MemTypes[4] = {"UC", "WC", "WT", "WB"}; // matches GMM_GFX_MEMORY_TYPE enum values
471
472 CachePolicyElement.Initialized = 1;
473
474 GMM_DPF(GFXDBG_NORMAL,
475 "Cache Policy Init: Degrading PAT settings to UC (uncached) from %s for Element %d\n",
476 MemTypes[GetWantedMemoryType(CachePolicyUsage)], Usage);
477
478 PATIdx = BestMatchingPATIdx(CachePolicyElement);
479 if(PATIdx == GMM_PAT_ERROR)
480 {
481 Success = false;
482 }
483 }
484 }
485 if(PATIdx != GMM_PAT_ERROR)
486 {
487 PTE.Gen8.PAT = (PATIdx & __BIT(2)) ? 1 : 0;
488 PTE.Gen8.PCD = (PATIdx & __BIT(1)) ? 1 : 0;
489 PTE.Gen8.PWT = (PATIdx & __BIT(0)) ? 1 : 0;
490 }
491 else
492 {
493 PTE.DwordValue = 0x0;
494 }
495 #else
496 GMM_UNREFERENCED_PARAMETER(CachePolicyUsage);
497 GMM_UNREFERENCED_PARAMETER(Usage);
498 #endif
499 *pPTEDwordValue = PTE.DwordValue;
500 return Success;
501 }
502