xref: /aosp_15_r20/external/gmmlib/Source/GmmLib/CachePolicy/GmmGen12CachePolicy.cpp (revision 35ffd701415c9e32e53136d61a677a8d0a8fc4a5)
1*35ffd701SAndroid Build Coastguard Worker /*==============================================================================
2*35ffd701SAndroid Build Coastguard Worker Copyright(c) 2019 Intel Corporation
3*35ffd701SAndroid Build Coastguard Worker 
4*35ffd701SAndroid Build Coastguard Worker Permission is hereby granted, free of charge, to any person obtaining a
5*35ffd701SAndroid Build Coastguard Worker copy of this software and associated documentation files(the "Software"),
6*35ffd701SAndroid Build Coastguard Worker to deal in the Software without restriction, including without limitation
7*35ffd701SAndroid Build Coastguard Worker the rights to use, copy, modify, merge, publish, distribute, sublicense,
8*35ffd701SAndroid Build Coastguard Worker and / or sell copies of the Software, and to permit persons to whom the
9*35ffd701SAndroid Build Coastguard Worker Software is furnished to do so, subject to the following conditions:
10*35ffd701SAndroid Build Coastguard Worker 
11*35ffd701SAndroid Build Coastguard Worker The above copyright notice and this permission notice shall be included
12*35ffd701SAndroid Build Coastguard Worker in all copies or substantial portions of the Software.
13*35ffd701SAndroid Build Coastguard Worker 
14*35ffd701SAndroid Build Coastguard Worker THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15*35ffd701SAndroid Build Coastguard Worker OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16*35ffd701SAndroid Build Coastguard Worker FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17*35ffd701SAndroid Build Coastguard Worker THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18*35ffd701SAndroid Build Coastguard Worker OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19*35ffd701SAndroid Build Coastguard Worker ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20*35ffd701SAndroid Build Coastguard Worker OTHER DEALINGS IN THE SOFTWARE.
21*35ffd701SAndroid Build Coastguard Worker ============================================================================*/
22*35ffd701SAndroid Build Coastguard Worker 
23*35ffd701SAndroid Build Coastguard Worker 
24*35ffd701SAndroid Build Coastguard Worker #include "Internal/Common/GmmLibInc.h"
25*35ffd701SAndroid Build Coastguard Worker #include "External/Common/GmmCachePolicy.h"
26*35ffd701SAndroid Build Coastguard Worker #include "External/Common/CachePolicy/GmmCachePolicyGen10.h"
27*35ffd701SAndroid Build Coastguard Worker #include "External/Common/CachePolicy/GmmCachePolicyGen11.h"
28*35ffd701SAndroid Build Coastguard Worker #include "External/Common/CachePolicy/GmmCachePolicyGen12.h"
29*35ffd701SAndroid Build Coastguard Worker 
30*35ffd701SAndroid Build Coastguard Worker //=============================================================================
31*35ffd701SAndroid Build Coastguard Worker //
32*35ffd701SAndroid Build Coastguard Worker // Function: IsSpecialMOCSUsage
33*35ffd701SAndroid Build Coastguard Worker //
34*35ffd701SAndroid Build Coastguard Worker // Desc: This function returns special(hw-reserved) MocsIdx based on usage
35*35ffd701SAndroid Build Coastguard Worker //
36*35ffd701SAndroid Build Coastguard Worker // Parameters: usage  -> Resource usage type
37*35ffd701SAndroid Build Coastguard Worker //             UpdateMOCS  -> True if MOCS Table must be updated, ow false
38*35ffd701SAndroid Build Coastguard Worker //
39*35ffd701SAndroid Build Coastguard Worker // Return: int32_t
40*35ffd701SAndroid Build Coastguard Worker //
41*35ffd701SAndroid Build Coastguard Worker //-----------------------------------------------------------------------------
IsSpecialMOCSUsage(GMM_RESOURCE_USAGE_TYPE Usage,bool & UpdateMOCS)42*35ffd701SAndroid Build Coastguard Worker int32_t GmmLib::GmmGen12CachePolicy::IsSpecialMOCSUsage(GMM_RESOURCE_USAGE_TYPE Usage, bool &UpdateMOCS)
43*35ffd701SAndroid Build Coastguard Worker {
44*35ffd701SAndroid Build Coastguard Worker     int32_t MocsIdx = -1;
45*35ffd701SAndroid Build Coastguard Worker     UpdateMOCS      = true;
46*35ffd701SAndroid Build Coastguard Worker //Macros for L3-Eviction Type
47*35ffd701SAndroid Build Coastguard Worker #define NA 0x0
48*35ffd701SAndroid Build Coastguard Worker #define RO 0x1
49*35ffd701SAndroid Build Coastguard Worker #define RW 0x2
50*35ffd701SAndroid Build Coastguard Worker #define SP 0x3
51*35ffd701SAndroid Build Coastguard Worker 
52*35ffd701SAndroid Build Coastguard Worker     switch(Usage)
53*35ffd701SAndroid Build Coastguard Worker     {
54*35ffd701SAndroid Build Coastguard Worker         case GMM_RESOURCE_USAGE_CCS:
55*35ffd701SAndroid Build Coastguard Worker             __GMM_ASSERT(pCachePolicy[Usage].L3 == 0); //Architecturally, CCS isn't L3-cacheable.
56*35ffd701SAndroid Build Coastguard Worker 
57*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].L3 = 0;
58*35ffd701SAndroid Build Coastguard Worker             MocsIdx                = 60;
59*35ffd701SAndroid Build Coastguard Worker             break;
60*35ffd701SAndroid Build Coastguard Worker         case GMM_RESOURCE_USAGE_MOCS_62:
61*35ffd701SAndroid Build Coastguard Worker             __GMM_ASSERT(pCachePolicy[Usage].L3 == 0); //Architecturally, TR/Aux-TT node isn't L3-cacheable.
62*35ffd701SAndroid Build Coastguard Worker 
63*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].L3 = 0;
64*35ffd701SAndroid Build Coastguard Worker             MocsIdx                = 62;
65*35ffd701SAndroid Build Coastguard Worker             break;
66*35ffd701SAndroid Build Coastguard Worker         case GMM_RESOURCE_USAGE_L3_EVICTION:
67*35ffd701SAndroid Build Coastguard Worker             __GMM_ASSERT(pCachePolicy[Usage].L3 == 0 &&
68*35ffd701SAndroid Build Coastguard Worker                          pCachePolicy[Usage].L3Eviction == RW); //Reserved MOCS for L3-evictions
69*35ffd701SAndroid Build Coastguard Worker 
70*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].L3         = 0;
71*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].L3Eviction = RW;
72*35ffd701SAndroid Build Coastguard Worker             MocsIdx                        = 63;
73*35ffd701SAndroid Build Coastguard Worker             break;
74*35ffd701SAndroid Build Coastguard Worker         case GMM_RESOURCE_USAGE_L3_EVICTION_SPECIAL:
75*35ffd701SAndroid Build Coastguard Worker         case GMM_RESOURCE_USAGE_CCS_MEDIA_WRITABLE:
76*35ffd701SAndroid Build Coastguard Worker             __GMM_ASSERT(pCachePolicy[Usage].L3 &&
77*35ffd701SAndroid Build Coastguard Worker                          pCachePolicy[Usage].L3Eviction == SP); //Reserved MOCS for L3-evictions
78*35ffd701SAndroid Build Coastguard Worker                                                                 //Special-case for Displayable, and similar non-LLC accesses
79*35ffd701SAndroid Build Coastguard Worker             GMM_ASSERTDPF(pCachePolicy[Usage].LLC == 0, "MOCS#61's Special Eviction isn't for LLC caching");
80*35ffd701SAndroid Build Coastguard Worker 
81*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].L3         = 1;
82*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].L3Eviction = SP;
83*35ffd701SAndroid Build Coastguard Worker             MocsIdx                        = 61;
84*35ffd701SAndroid Build Coastguard Worker             break;
85*35ffd701SAndroid Build Coastguard Worker         default:
86*35ffd701SAndroid Build Coastguard Worker             UpdateMOCS = false;
87*35ffd701SAndroid Build Coastguard Worker             break;
88*35ffd701SAndroid Build Coastguard Worker     }
89*35ffd701SAndroid Build Coastguard Worker 
90*35ffd701SAndroid Build Coastguard Worker     if(pCachePolicy[Usage].L3Eviction == RW)
91*35ffd701SAndroid Build Coastguard Worker     {
92*35ffd701SAndroid Build Coastguard Worker         GMM_CACHE_POLICY_ELEMENT L3Eviction;
93*35ffd701SAndroid Build Coastguard Worker         L3Eviction.Value = pCachePolicy[GMM_RESOURCE_USAGE_L3_EVICTION].Value;
94*35ffd701SAndroid Build Coastguard Worker 
95*35ffd701SAndroid Build Coastguard Worker         //For internal purpose, hw overrides MOCS#63 as L3-uncacheable, still using it for L3-evictions
96*35ffd701SAndroid Build Coastguard Worker         if(Usage != GMM_RESOURCE_USAGE_L3_EVICTION)
97*35ffd701SAndroid Build Coastguard Worker         {
98*35ffd701SAndroid Build Coastguard Worker             L3Eviction.L3 = 1; //Override L3, to verify MOCS#63 applicable or not
99*35ffd701SAndroid Build Coastguard Worker         }
100*35ffd701SAndroid Build Coastguard Worker 
101*35ffd701SAndroid Build Coastguard Worker         __GMM_ASSERT(pCachePolicy[Usage].Value == L3Eviction.Value); //Allow mis-match due to override registries
102*35ffd701SAndroid Build Coastguard Worker         //MocsIdx = 63;    //Use non-#63 MOCS, #63 itself is L3-uncached
103*35ffd701SAndroid Build Coastguard Worker     }
104*35ffd701SAndroid Build Coastguard Worker     else if(pCachePolicy[Usage].L3Eviction == SP)
105*35ffd701SAndroid Build Coastguard Worker     {
106*35ffd701SAndroid Build Coastguard Worker         __GMM_ASSERT(pCachePolicy[Usage].Value == pCachePolicy[GMM_RESOURCE_USAGE_L3_EVICTION_SPECIAL].Value); //Allow mis-match due to override registries
107*35ffd701SAndroid Build Coastguard Worker         MocsIdx = 61;
108*35ffd701SAndroid Build Coastguard Worker     }
109*35ffd701SAndroid Build Coastguard Worker 
110*35ffd701SAndroid Build Coastguard Worker     return MocsIdx;
111*35ffd701SAndroid Build Coastguard Worker }
112*35ffd701SAndroid Build Coastguard Worker //=============================================================================
113*35ffd701SAndroid Build Coastguard Worker //
114*35ffd701SAndroid Build Coastguard Worker // Function: __GmmGen12InitCachePolicy
115*35ffd701SAndroid Build Coastguard Worker //
116*35ffd701SAndroid Build Coastguard Worker // Desc: This function initializes the cache policy
117*35ffd701SAndroid Build Coastguard Worker //
118*35ffd701SAndroid Build Coastguard Worker // Parameters: pCachePolicy  -> Ptr to array to be populated with the
119*35ffd701SAndroid Build Coastguard Worker //             mapping of usages -> cache settings.
120*35ffd701SAndroid Build Coastguard Worker //
121*35ffd701SAndroid Build Coastguard Worker // Return: GMM_STATUS
122*35ffd701SAndroid Build Coastguard Worker //
123*35ffd701SAndroid Build Coastguard Worker //-----------------------------------------------------------------------------
InitCachePolicy()124*35ffd701SAndroid Build Coastguard Worker GMM_STATUS GmmLib::GmmGen12CachePolicy::InitCachePolicy()
125*35ffd701SAndroid Build Coastguard Worker {
126*35ffd701SAndroid Build Coastguard Worker 
127*35ffd701SAndroid Build Coastguard Worker     __GMM_ASSERTPTR(pCachePolicy, GMM_ERROR);
128*35ffd701SAndroid Build Coastguard Worker 
129*35ffd701SAndroid Build Coastguard Worker #define DEFINE_CACHE_ELEMENT(usage, llc, ellc, l3, wt, age, aom, lecc_scc, l3_scc, scf, sso, cos, hdcl1, l3evict) DEFINE_CP_ELEMENT(usage, llc, ellc, l3, wt, age, aom, lecc_scc, l3_scc, scf, sso, cos, hdcl1, l3evict, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
130*35ffd701SAndroid Build Coastguard Worker 
131*35ffd701SAndroid Build Coastguard Worker #include "GmmGen12CachePolicy.h"
132*35ffd701SAndroid Build Coastguard Worker 
133*35ffd701SAndroid Build Coastguard Worker #define TC_LLC (1)
134*35ffd701SAndroid Build Coastguard Worker #define TC_ELLC (0)
135*35ffd701SAndroid Build Coastguard Worker #define TC_LLC_ELLC (2)
136*35ffd701SAndroid Build Coastguard Worker 
137*35ffd701SAndroid Build Coastguard Worker #define LeCC_UNCACHEABLE (0x0)
138*35ffd701SAndroid Build Coastguard Worker #define LeCC_WC_UNCACHEABLE (0x1)
139*35ffd701SAndroid Build Coastguard Worker #define LeCC_WT_CACHEABLE (0x2) //Only used as MemPushWRite disqualifier if set along with eLLC-only -still holds on gen12+?
140*35ffd701SAndroid Build Coastguard Worker #define LeCC_WB_CACHEABLE (0x3)
141*35ffd701SAndroid Build Coastguard Worker 
142*35ffd701SAndroid Build Coastguard Worker #define L3_UNCACHEABLE (0x1)
143*35ffd701SAndroid Build Coastguard Worker #define L3_WB_CACHEABLE (0x3)
144*35ffd701SAndroid Build Coastguard Worker 
145*35ffd701SAndroid Build Coastguard Worker #define DISABLE_SKIP_CACHING_CONTROL (0x0)
146*35ffd701SAndroid Build Coastguard Worker #define ENABLE_SKIP_CACHING_CONTROL (0x1)
147*35ffd701SAndroid Build Coastguard Worker 
148*35ffd701SAndroid Build Coastguard Worker #define DISABLE_SELF_SNOOP_OVERRIDE (0x0)
149*35ffd701SAndroid Build Coastguard Worker #define ENABLE_SELF_SNOOP_OVERRIDE (0x1)
150*35ffd701SAndroid Build Coastguard Worker #define ENABLE_SELF_SNOOP_ALWAYS (0x3)
151*35ffd701SAndroid Build Coastguard Worker #define CLASS_SERVICE_ZERO (0x0)
152*35ffd701SAndroid Build Coastguard Worker 
153*35ffd701SAndroid Build Coastguard Worker     {
154*35ffd701SAndroid Build Coastguard Worker         SetUpMOCSTable();
155*35ffd701SAndroid Build Coastguard Worker     }
156*35ffd701SAndroid Build Coastguard Worker 
157*35ffd701SAndroid Build Coastguard Worker     {
158*35ffd701SAndroid Build Coastguard Worker         // Define index of cache element
159*35ffd701SAndroid Build Coastguard Worker         uint32_t Usage = 0;
160*35ffd701SAndroid Build Coastguard Worker 
161*35ffd701SAndroid Build Coastguard Worker #if(_WIN32 && (_DEBUG || _RELEASE_INTERNAL))
162*35ffd701SAndroid Build Coastguard Worker         void *pKmdGmmContext = NULL;
163*35ffd701SAndroid Build Coastguard Worker 
164*35ffd701SAndroid Build Coastguard Worker         OverrideCachePolicy(pKmdGmmContext);
165*35ffd701SAndroid Build Coastguard Worker #endif
166*35ffd701SAndroid Build Coastguard Worker         // Process the cache policy and fill in the look up table
167*35ffd701SAndroid Build Coastguard Worker         for(; Usage < GMM_RESOURCE_USAGE_MAX; Usage++)
168*35ffd701SAndroid Build Coastguard Worker         {
169*35ffd701SAndroid Build Coastguard Worker             bool                         CachePolicyError = false;
170*35ffd701SAndroid Build Coastguard Worker             bool                         SpecialMOCS      = false;
171*35ffd701SAndroid Build Coastguard Worker             int32_t                      CPTblIdx         = -1;
172*35ffd701SAndroid Build Coastguard Worker             uint32_t                     j                = 0;
173*35ffd701SAndroid Build Coastguard Worker             uint64_t                     PTEValue         = 0;
174*35ffd701SAndroid Build Coastguard Worker             GMM_CACHE_POLICY_TBL_ELEMENT UsageEle         = {0};
175*35ffd701SAndroid Build Coastguard Worker 
176*35ffd701SAndroid Build Coastguard Worker             CPTblIdx = IsSpecialMOCSUsage((GMM_RESOURCE_USAGE_TYPE)Usage, SpecialMOCS);
177*35ffd701SAndroid Build Coastguard Worker 
178*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.Reserved = 0; // Reserved bits zeroe'd, this is so we
179*35ffd701SAndroid Build Coastguard Worker                                         // we can compare the unioned LeCC.DwordValue.
180*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.SelfSnoop = DISABLE_SELF_SNOOP_OVERRIDE;
181*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.CoS       = CLASS_SERVICE_ZERO;
182*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.SCC       = 0;
183*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.ESC       = 0;
184*35ffd701SAndroid Build Coastguard Worker 
185*35ffd701SAndroid Build Coastguard Worker             if(pCachePolicy[Usage].SCF && pGmmLibContext->GetSkuTable().FtrLLCBypass)
186*35ffd701SAndroid Build Coastguard Worker             {
187*35ffd701SAndroid Build Coastguard Worker                 UsageEle.LeCC.SCF = pCachePolicy[Usage].SCF;
188*35ffd701SAndroid Build Coastguard Worker                 __GMM_ASSERT(pCachePolicy[Usage].LLC == 0); //LLC and ByPassLLC are mutually-exclusive
189*35ffd701SAndroid Build Coastguard Worker             }
190*35ffd701SAndroid Build Coastguard Worker 
191*35ffd701SAndroid Build Coastguard Worker             if(pCachePolicy[Usage].SSO & ENABLE_SELF_SNOOP_OVERRIDE)
192*35ffd701SAndroid Build Coastguard Worker             {
193*35ffd701SAndroid Build Coastguard Worker                 UsageEle.LeCC.SelfSnoop = pCachePolicy[Usage].SSO & ENABLE_SELF_SNOOP_ALWAYS;
194*35ffd701SAndroid Build Coastguard Worker             }
195*35ffd701SAndroid Build Coastguard Worker             if(pCachePolicy[Usage].CoS)
196*35ffd701SAndroid Build Coastguard Worker             {
197*35ffd701SAndroid Build Coastguard Worker                 UsageEle.LeCC.CoS = pCachePolicy[Usage].CoS;
198*35ffd701SAndroid Build Coastguard Worker             }
199*35ffd701SAndroid Build Coastguard Worker             if(pCachePolicy[Usage].HDCL1)
200*35ffd701SAndroid Build Coastguard Worker             {
201*35ffd701SAndroid Build Coastguard Worker                 UsageEle.HDCL1 = 1;
202*35ffd701SAndroid Build Coastguard Worker             }
203*35ffd701SAndroid Build Coastguard Worker             if(pCachePolicy[Usage].LeCC_SCC)
204*35ffd701SAndroid Build Coastguard Worker             {
205*35ffd701SAndroid Build Coastguard Worker                 UsageEle.LeCC.SCC = pCachePolicy[Usage].LeCC_SCC;
206*35ffd701SAndroid Build Coastguard Worker                 UsageEle.LeCC.ESC = ENABLE_SKIP_CACHING_CONTROL;
207*35ffd701SAndroid Build Coastguard Worker             }
208*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.LRUM = pCachePolicy[Usage].AGE;
209*35ffd701SAndroid Build Coastguard Worker 
210*35ffd701SAndroid Build Coastguard Worker             // default to LLC target cache.
211*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.TargetCache  = TC_LLC;
212*35ffd701SAndroid Build Coastguard Worker             UsageEle.LeCC.Cacheability = LeCC_WB_CACHEABLE;
213*35ffd701SAndroid Build Coastguard Worker             if(pCachePolicy[Usage].LLC)
214*35ffd701SAndroid Build Coastguard Worker             {
215*35ffd701SAndroid Build Coastguard Worker                 UsageEle.LeCC.TargetCache = TC_LLC;
216*35ffd701SAndroid Build Coastguard Worker                 __GMM_ASSERT(pCachePolicy[Usage].SCF == 0); //LLC and ByPassLLC are mutually-exclusive
217*35ffd701SAndroid Build Coastguard Worker             }
218*35ffd701SAndroid Build Coastguard Worker             else
219*35ffd701SAndroid Build Coastguard Worker             {
220*35ffd701SAndroid Build Coastguard Worker                 UsageEle.LeCC.Cacheability = LeCC_WC_UNCACHEABLE;
221*35ffd701SAndroid Build Coastguard Worker             }
222*35ffd701SAndroid Build Coastguard Worker             UsageEle.L3.Reserved = 0; // Reserved bits zeroe'd, this is so we
223*35ffd701SAndroid Build Coastguard Worker                                       // we can compare the unioned L3.UshortValue.
224*35ffd701SAndroid Build Coastguard Worker             UsageEle.L3.ESC          = DISABLE_SKIP_CACHING_CONTROL;
225*35ffd701SAndroid Build Coastguard Worker             UsageEle.L3.SCC          = 0;
226*35ffd701SAndroid Build Coastguard Worker             UsageEle.L3.Cacheability = pCachePolicy[Usage].L3 ? L3_WB_CACHEABLE : L3_UNCACHEABLE;
227*35ffd701SAndroid Build Coastguard Worker 
228*35ffd701SAndroid Build Coastguard Worker             __GMM_ASSERT((pCachePolicy[Usage].L3 && pCachePolicy[Usage].L3Eviction != 0) ||
229*35ffd701SAndroid Build Coastguard Worker                          (pCachePolicy[Usage].L3 == 0 && (pCachePolicy[Usage].L3Eviction == 0 || Usage == GMM_RESOURCE_USAGE_L3_EVICTION)));
230*35ffd701SAndroid Build Coastguard Worker 
231*35ffd701SAndroid Build Coastguard Worker             if(pCachePolicy[Usage].L3_SCC)
232*35ffd701SAndroid Build Coastguard Worker             {
233*35ffd701SAndroid Build Coastguard Worker                 UsageEle.L3.ESC = ENABLE_SKIP_CACHING_CONTROL;
234*35ffd701SAndroid Build Coastguard Worker                 UsageEle.L3.SCC = (uint16_t)pCachePolicy[Usage].L3_SCC;
235*35ffd701SAndroid Build Coastguard Worker             }
236*35ffd701SAndroid Build Coastguard Worker 
237*35ffd701SAndroid Build Coastguard Worker             //Special-case MOCS handling for MOCS Table Index 60-63
238*35ffd701SAndroid Build Coastguard Worker             if(CPTblIdx >= GMM_GEN12_MAX_NUMBER_MOCS_INDEXES)
239*35ffd701SAndroid Build Coastguard Worker             {
240*35ffd701SAndroid Build Coastguard Worker                 GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &pGmmLibContext->GetCachePolicyTlbElement()[CPTblIdx];
241*35ffd701SAndroid Build Coastguard Worker 
242*35ffd701SAndroid Build Coastguard Worker                 if(SpecialMOCS &&
243*35ffd701SAndroid Build Coastguard Worker                    !(TblEle->LeCC.DwordValue == UsageEle.LeCC.DwordValue &&
244*35ffd701SAndroid Build Coastguard Worker                      TblEle->L3.UshortValue == UsageEle.L3.UshortValue &&
245*35ffd701SAndroid Build Coastguard Worker                      TblEle->HDCL1 == UsageEle.HDCL1))
246*35ffd701SAndroid Build Coastguard Worker                 {
247*35ffd701SAndroid Build Coastguard Worker                     //Assert if being overwritten!
248*35ffd701SAndroid Build Coastguard Worker                     __GMM_ASSERT(TblEle->LeCC.DwordValue == 0 &&
249*35ffd701SAndroid Build Coastguard Worker                                  TblEle->L3.UshortValue == 0 &&
250*35ffd701SAndroid Build Coastguard Worker                                  TblEle->HDCL1 == 0);
251*35ffd701SAndroid Build Coastguard Worker 
252*35ffd701SAndroid Build Coastguard Worker #if(_WIN32 && (_DEBUG || _RELEASE_INTERNAL))
253*35ffd701SAndroid Build Coastguard Worker                     if(pCachePolicy[Usage].IsOverridenByRegkey)
254*35ffd701SAndroid Build Coastguard Worker                     {
255*35ffd701SAndroid Build Coastguard Worker                         TblEle->LeCC.DwordValue = UsageEle.LeCC.DwordValue;
256*35ffd701SAndroid Build Coastguard Worker                         TblEle->L3.UshortValue  = UsageEle.L3.UshortValue;
257*35ffd701SAndroid Build Coastguard Worker                         TblEle->HDCL1           = UsageEle.HDCL1;
258*35ffd701SAndroid Build Coastguard Worker                     }
259*35ffd701SAndroid Build Coastguard Worker #endif
260*35ffd701SAndroid Build Coastguard Worker                 }
261*35ffd701SAndroid Build Coastguard Worker             }
262*35ffd701SAndroid Build Coastguard Worker             //For HDC L1 caching, MOCS Table index 48-59 should be used
263*35ffd701SAndroid Build Coastguard Worker             else if(UsageEle.HDCL1)
264*35ffd701SAndroid Build Coastguard Worker             {
265*35ffd701SAndroid Build Coastguard Worker                 for(j = GMM_GEN10_HDCL1_MOCS_INDEX_START; j <= CurrentMaxL1HdcMocsIndex; j++)
266*35ffd701SAndroid Build Coastguard Worker                 {
267*35ffd701SAndroid Build Coastguard Worker                     GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &pGmmLibContext->GetCachePolicyTlbElement()[j];
268*35ffd701SAndroid Build Coastguard Worker                     if(TblEle->LeCC.DwordValue == UsageEle.LeCC.DwordValue &&
269*35ffd701SAndroid Build Coastguard Worker                        TblEle->L3.UshortValue == UsageEle.L3.UshortValue &&
270*35ffd701SAndroid Build Coastguard Worker                        TblEle->HDCL1 == UsageEle.HDCL1)
271*35ffd701SAndroid Build Coastguard Worker                     {
272*35ffd701SAndroid Build Coastguard Worker                         CPTblIdx = j;
273*35ffd701SAndroid Build Coastguard Worker                         break;
274*35ffd701SAndroid Build Coastguard Worker                     }
275*35ffd701SAndroid Build Coastguard Worker                 }
276*35ffd701SAndroid Build Coastguard Worker             }
277*35ffd701SAndroid Build Coastguard Worker             else
278*35ffd701SAndroid Build Coastguard Worker             {
279*35ffd701SAndroid Build Coastguard Worker                 // Due to unstable system behavior on TGLLP, MOCS #0 index had to be programmed as UC in MOCS lookup table - pCachePolicyTlbElement
280*35ffd701SAndroid Build Coastguard Worker                 // But still Index 0 is Reserved for Error by HW and should not be used.
281*35ffd701SAndroid Build Coastguard Worker                 // Hence Gmmlib will opt out from the MOCS#0 usage and Lookup into MOCS table and MOCS index assigment must start from Index 1.
282*35ffd701SAndroid Build Coastguard Worker                 for(j = 1; j <= CurrentMaxMocsIndex; j++)
283*35ffd701SAndroid Build Coastguard Worker                 {
284*35ffd701SAndroid Build Coastguard Worker                     GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &pGmmLibContext->GetCachePolicyTlbElement()[j];
285*35ffd701SAndroid Build Coastguard Worker                     if(TblEle->LeCC.DwordValue == UsageEle.LeCC.DwordValue &&
286*35ffd701SAndroid Build Coastguard Worker                        TblEle->L3.UshortValue == UsageEle.L3.UshortValue &&
287*35ffd701SAndroid Build Coastguard Worker                        TblEle->HDCL1 == UsageEle.HDCL1)
288*35ffd701SAndroid Build Coastguard Worker                     {
289*35ffd701SAndroid Build Coastguard Worker                         CPTblIdx = j;
290*35ffd701SAndroid Build Coastguard Worker                         break;
291*35ffd701SAndroid Build Coastguard Worker                     }
292*35ffd701SAndroid Build Coastguard Worker                 }
293*35ffd701SAndroid Build Coastguard Worker             }
294*35ffd701SAndroid Build Coastguard Worker 
295*35ffd701SAndroid Build Coastguard Worker             // Didn't find the caching settings in one of the already programmed lookup table entries.
296*35ffd701SAndroid Build Coastguard Worker             // Need to add a new lookup table entry.
297*35ffd701SAndroid Build Coastguard Worker             if(CPTblIdx == -1)
298*35ffd701SAndroid Build Coastguard Worker             {
299*35ffd701SAndroid Build Coastguard Worker 
300*35ffd701SAndroid Build Coastguard Worker #if(_WIN32 && (_DEBUG || _RELEASE_INTERNAL))
301*35ffd701SAndroid Build Coastguard Worker                 // If the Cache Policy setting is overriden through regkey,
302*35ffd701SAndroid Build Coastguard Worker                 // don't raise an assert/log error. Raising an assert for debug/perf testing isn't really helpful
303*35ffd701SAndroid Build Coastguard Worker                 if(pCachePolicy[Usage].IsOverridenByRegkey)
304*35ffd701SAndroid Build Coastguard Worker                 {
305*35ffd701SAndroid Build Coastguard Worker                     if(UsageEle.HDCL1 && CurrentMaxL1HdcMocsIndex < GMM_GEN12_MAX_NUMBER_MOCS_INDEXES - 1)
306*35ffd701SAndroid Build Coastguard Worker                     {
307*35ffd701SAndroid Build Coastguard Worker                         GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &(pGmmLibContext->GetCachePolicyTlbElement()[++CurrentMaxL1HdcMocsIndex]);
308*35ffd701SAndroid Build Coastguard Worker                         CPTblIdx                             = CurrentMaxL1HdcMocsIndex;
309*35ffd701SAndroid Build Coastguard Worker 
310*35ffd701SAndroid Build Coastguard Worker                         TblEle->LeCC.DwordValue = UsageEle.LeCC.DwordValue;
311*35ffd701SAndroid Build Coastguard Worker                         TblEle->L3.UshortValue  = UsageEle.L3.UshortValue;
312*35ffd701SAndroid Build Coastguard Worker                         TblEle->HDCL1           = UsageEle.HDCL1;
313*35ffd701SAndroid Build Coastguard Worker                     }
314*35ffd701SAndroid Build Coastguard Worker                     else if(CurrentMaxMocsIndex < GMM_GEN10_HDCL1_MOCS_INDEX_START)
315*35ffd701SAndroid Build Coastguard Worker                     {
316*35ffd701SAndroid Build Coastguard Worker                         GMM_CACHE_POLICY_TBL_ELEMENT *TblEle = &(pGmmLibContext->GetCachePolicyTlbElement()[++CurrentMaxMocsIndex]);
317*35ffd701SAndroid Build Coastguard Worker                         CPTblIdx                             = CurrentMaxMocsIndex;
318*35ffd701SAndroid Build Coastguard Worker 
319*35ffd701SAndroid Build Coastguard Worker                         TblEle->LeCC.DwordValue = UsageEle.LeCC.DwordValue;
320*35ffd701SAndroid Build Coastguard Worker                         TblEle->L3.UshortValue  = UsageEle.L3.UshortValue;
321*35ffd701SAndroid Build Coastguard Worker                         TblEle->HDCL1           = UsageEle.HDCL1;
322*35ffd701SAndroid Build Coastguard Worker                     }
323*35ffd701SAndroid Build Coastguard Worker                     else
324*35ffd701SAndroid Build Coastguard Worker                     {
325*35ffd701SAndroid Build Coastguard Worker                         // Too many unique caching combinations to program the
326*35ffd701SAndroid Build Coastguard Worker                         // MOCS lookup table.
327*35ffd701SAndroid Build Coastguard Worker                         CachePolicyError = true;
328*35ffd701SAndroid Build Coastguard Worker                         GMM_ASSERTDPF(
329*35ffd701SAndroid Build Coastguard Worker                         "Cache Policy Init Error: Invalid Cache Programming, too many unique caching combinations"
330*35ffd701SAndroid Build Coastguard Worker                         "(we only support GMM_GEN_MAX_NUMBER_MOCS_INDEXES = %d)",
331*35ffd701SAndroid Build Coastguard Worker                         GMM_MAX_NUMBER_MOCS_INDEXES - 1);
332*35ffd701SAndroid Build Coastguard Worker                         // Set cache policy index to uncached.
333*35ffd701SAndroid Build Coastguard Worker                         CPTblIdx = 3;
334*35ffd701SAndroid Build Coastguard Worker                     }
335*35ffd701SAndroid Build Coastguard Worker                 }
336*35ffd701SAndroid Build Coastguard Worker                 else
337*35ffd701SAndroid Build Coastguard Worker #endif
338*35ffd701SAndroid Build Coastguard Worker                 {
339*35ffd701SAndroid Build Coastguard Worker                     GMM_ASSERTDPF(false, "CRITICAL ERROR: Cache Policy Usage value specified by Client is not defined in Fixed MOCS Table!");
340*35ffd701SAndroid Build Coastguard Worker 
341*35ffd701SAndroid Build Coastguard Worker                     CachePolicyError = true;
342*35ffd701SAndroid Build Coastguard Worker                     GMM_ASSERTDPF(
343*35ffd701SAndroid Build Coastguard Worker                     "Cache Policy Init Error: Invalid Cache Programming, too many unique caching combinations"
344*35ffd701SAndroid Build Coastguard Worker                     "(we only support GMM_GEN_MAX_NUMBER_MOCS_INDEXES = %d)",
345*35ffd701SAndroid Build Coastguard Worker                     CurrentMaxMocsIndex);
346*35ffd701SAndroid Build Coastguard Worker 
347*35ffd701SAndroid Build Coastguard Worker                     // Set cache policy index to uncached.
348*35ffd701SAndroid Build Coastguard Worker                     CPTblIdx = 3;
349*35ffd701SAndroid Build Coastguard Worker                 }
350*35ffd701SAndroid Build Coastguard Worker             }
351*35ffd701SAndroid Build Coastguard Worker 
352*35ffd701SAndroid Build Coastguard Worker             // PTE entries do not control caching on SKL+ (for legacy context)
353*35ffd701SAndroid Build Coastguard Worker             if(!GetUsagePTEValue(pCachePolicy[Usage], Usage, &PTEValue))
354*35ffd701SAndroid Build Coastguard Worker             {
355*35ffd701SAndroid Build Coastguard Worker                 CachePolicyError = true;
356*35ffd701SAndroid Build Coastguard Worker             }
357*35ffd701SAndroid Build Coastguard Worker 
358*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].PTE.DwordValue     = PTEValue & 0xFFFFFFFF;
359*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].PTE.HighDwordValue = 0;
360*35ffd701SAndroid Build Coastguard Worker 
361*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].MemoryObjectOverride.Gen12.Index = CPTblIdx;
362*35ffd701SAndroid Build Coastguard Worker 
363*35ffd701SAndroid Build Coastguard Worker             pCachePolicy[Usage].Override = ALWAYS_OVERRIDE;
364*35ffd701SAndroid Build Coastguard Worker 
365*35ffd701SAndroid Build Coastguard Worker             if(CachePolicyError)
366*35ffd701SAndroid Build Coastguard Worker             {
367*35ffd701SAndroid Build Coastguard Worker                 GMM_ASSERTDPF("Cache Policy Init Error: Invalid Cache Programming - Element %d", Usage);
368*35ffd701SAndroid Build Coastguard Worker             }
369*35ffd701SAndroid Build Coastguard Worker         }
370*35ffd701SAndroid Build Coastguard Worker     }
371*35ffd701SAndroid Build Coastguard Worker 
372*35ffd701SAndroid Build Coastguard Worker     return GMM_SUCCESS;
373*35ffd701SAndroid Build Coastguard Worker }
374*35ffd701SAndroid Build Coastguard Worker 
375*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
376*35ffd701SAndroid Build Coastguard Worker /// Return true if (MT2) is a better match for (WantedMT)
377*35ffd701SAndroid Build Coastguard Worker ///       than (MT1)
378*35ffd701SAndroid Build Coastguard Worker ///
379*35ffd701SAndroid Build Coastguard Worker /// @param[in]         WantedMT: Wanted Memory Type
380*35ffd701SAndroid Build Coastguard Worker /// @param[in]         MT1: Memory Type for PATIdx1
381*35ffd701SAndroid Build Coastguard Worker /// @param[in]         MT2: Memory Type for PATIdx2
382*35ffd701SAndroid Build Coastguard Worker ///
383*35ffd701SAndroid Build Coastguard Worker /// @return            Select the new PAT Index True/False
384*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
SelectNewPATIdx(GMM_GFX_MEMORY_TYPE WantedMT,GMM_GFX_MEMORY_TYPE MT1,GMM_GFX_MEMORY_TYPE MT2)385*35ffd701SAndroid Build Coastguard Worker uint8_t GmmLib::GmmGen12CachePolicy::SelectNewPATIdx(GMM_GFX_MEMORY_TYPE WantedMT,
386*35ffd701SAndroid Build Coastguard Worker                                                      GMM_GFX_MEMORY_TYPE MT1, GMM_GFX_MEMORY_TYPE MT2)
387*35ffd701SAndroid Build Coastguard Worker {
388*35ffd701SAndroid Build Coastguard Worker     uint8_t SelectPAT2 = 0;
389*35ffd701SAndroid Build Coastguard Worker 
390*35ffd701SAndroid Build Coastguard Worker     // select on Memory Type
391*35ffd701SAndroid Build Coastguard Worker     if(MT1 != WantedMT)
392*35ffd701SAndroid Build Coastguard Worker     {
393*35ffd701SAndroid Build Coastguard Worker         if(MT2 == WantedMT || MT2 == GMM_GFX_UC_WITH_FENCE)
394*35ffd701SAndroid Build Coastguard Worker         {
395*35ffd701SAndroid Build Coastguard Worker             SelectPAT2 = 1;
396*35ffd701SAndroid Build Coastguard Worker         }
397*35ffd701SAndroid Build Coastguard Worker         goto EXIT;
398*35ffd701SAndroid Build Coastguard Worker     }
399*35ffd701SAndroid Build Coastguard Worker 
400*35ffd701SAndroid Build Coastguard Worker EXIT:
401*35ffd701SAndroid Build Coastguard Worker     return SelectPAT2;
402*35ffd701SAndroid Build Coastguard Worker }
403*35ffd701SAndroid Build Coastguard Worker 
404*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
405*35ffd701SAndroid Build Coastguard Worker /// Returns the PAT idx that best matches the cache policy for this usage.
406*35ffd701SAndroid Build Coastguard Worker ///
407*35ffd701SAndroid Build Coastguard Worker /// @param: CachePolicy: cache policy for a usage
408*35ffd701SAndroid Build Coastguard Worker ///
409*35ffd701SAndroid Build Coastguard Worker /// @return        PAT Idx to use in the PTE
410*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
BestMatchingPATIdx(GMM_CACHE_POLICY_ELEMENT CachePolicy)411*35ffd701SAndroid Build Coastguard Worker uint32_t GmmLib::GmmGen12CachePolicy::BestMatchingPATIdx(GMM_CACHE_POLICY_ELEMENT CachePolicy)
412*35ffd701SAndroid Build Coastguard Worker {
413*35ffd701SAndroid Build Coastguard Worker     uint32_t            i;
414*35ffd701SAndroid Build Coastguard Worker     uint32_t            PATIdx           = 0;
415*35ffd701SAndroid Build Coastguard Worker     GMM_GFX_MEMORY_TYPE WantedMemoryType = GMM_GFX_UC_WITH_FENCE, MemoryType;
416*35ffd701SAndroid Build Coastguard Worker     WA_TABLE *          pWaTable         = &const_cast<WA_TABLE &>(pGmmLibContext->GetWaTable());
417*35ffd701SAndroid Build Coastguard Worker 
418*35ffd701SAndroid Build Coastguard Worker     WantedMemoryType = GetWantedMemoryType(CachePolicy);
419*35ffd701SAndroid Build Coastguard Worker 
420*35ffd701SAndroid Build Coastguard Worker     // Override wantedMemoryType so that PAT.MT is UC
421*35ffd701SAndroid Build Coastguard Worker     // Gen12 uses max function to resolve PAT-vs-MOCS MemType, So unless PTE.PAT says UC, MOCS won't be able to set UC!
422*35ffd701SAndroid Build Coastguard Worker     if(pWaTable->WaMemTypeIsMaxOfPatAndMocs)
423*35ffd701SAndroid Build Coastguard Worker     {
424*35ffd701SAndroid Build Coastguard Worker         WantedMemoryType = GMM_GFX_UC_WITH_FENCE;
425*35ffd701SAndroid Build Coastguard Worker     }
426*35ffd701SAndroid Build Coastguard Worker 
427*35ffd701SAndroid Build Coastguard Worker     for(i = 1; i < NumPATRegisters; i++)
428*35ffd701SAndroid Build Coastguard Worker     {
429*35ffd701SAndroid Build Coastguard Worker         GMM_PRIVATE_PAT PAT1 = GetPrivatePATEntry(PATIdx);
430*35ffd701SAndroid Build Coastguard Worker         GMM_PRIVATE_PAT PAT2 = GetPrivatePATEntry(i);
431*35ffd701SAndroid Build Coastguard Worker 
432*35ffd701SAndroid Build Coastguard Worker         if(SelectNewPATIdx(WantedMemoryType,
433*35ffd701SAndroid Build Coastguard Worker                            (GMM_GFX_MEMORY_TYPE)PAT1.Gen12.MemoryType,
434*35ffd701SAndroid Build Coastguard Worker                            (GMM_GFX_MEMORY_TYPE)PAT2.Gen12.MemoryType))
435*35ffd701SAndroid Build Coastguard Worker         {
436*35ffd701SAndroid Build Coastguard Worker             PATIdx = i;
437*35ffd701SAndroid Build Coastguard Worker         }
438*35ffd701SAndroid Build Coastguard Worker     }
439*35ffd701SAndroid Build Coastguard Worker 
440*35ffd701SAndroid Build Coastguard Worker     MemoryType = (GMM_GFX_MEMORY_TYPE)GetPrivatePATEntry(PATIdx).Gen12.MemoryType;
441*35ffd701SAndroid Build Coastguard Worker 
442*35ffd701SAndroid Build Coastguard Worker     if(MemoryType != WantedMemoryType)
443*35ffd701SAndroid Build Coastguard Worker     {
444*35ffd701SAndroid Build Coastguard Worker         // Failed to find a matching PAT entry
445*35ffd701SAndroid Build Coastguard Worker         return GMM_PAT_ERROR;
446*35ffd701SAndroid Build Coastguard Worker     }
447*35ffd701SAndroid Build Coastguard Worker     return PATIdx;
448*35ffd701SAndroid Build Coastguard Worker }
449*35ffd701SAndroid Build Coastguard Worker 
450*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
451*35ffd701SAndroid Build Coastguard Worker /// Initializes WA's needed for setting up the Private PATs
452*35ffd701SAndroid Build Coastguard Worker /// WaNoMocsEllcOnly (reset)
453*35ffd701SAndroid Build Coastguard Worker /// WaGttPat0, WaGttPat0GttWbOverOsIommuEllcOnly, WaGttPat0WB (use from base class)
454*35ffd701SAndroid Build Coastguard Worker ///
455*35ffd701SAndroid Build Coastguard Worker /// @return        GMM_STATUS
456*35ffd701SAndroid Build Coastguard Worker ///
457*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
SetPATInitWA()458*35ffd701SAndroid Build Coastguard Worker GMM_STATUS GmmLib::GmmGen12CachePolicy::SetPATInitWA()
459*35ffd701SAndroid Build Coastguard Worker {
460*35ffd701SAndroid Build Coastguard Worker     GMM_STATUS Status   = GMM_SUCCESS;
461*35ffd701SAndroid Build Coastguard Worker     WA_TABLE * pWaTable = &const_cast<WA_TABLE &>(pGmmLibContext->GetWaTable());
462*35ffd701SAndroid Build Coastguard Worker 
463*35ffd701SAndroid Build Coastguard Worker #if(defined(__GMM_KMD__))
464*35ffd701SAndroid Build Coastguard Worker     __GMM_ASSERT(pGmmLibContext->GetSkuTable().FtrMemTypeMocsDeferPAT == 0x0); //MOCS.TargetCache supports eLLC only, PAT.TC -> reserved bits.
465*35ffd701SAndroid Build Coastguard Worker     pWaTable->WaGttPat0WB = 0;                                                 //Override PAT #0
466*35ffd701SAndroid Build Coastguard Worker #else
467*35ffd701SAndroid Build Coastguard Worker     Status = GMM_ERROR;
468*35ffd701SAndroid Build Coastguard Worker #endif
469*35ffd701SAndroid Build Coastguard Worker 
470*35ffd701SAndroid Build Coastguard Worker     return Status;
471*35ffd701SAndroid Build Coastguard Worker }
472*35ffd701SAndroid Build Coastguard Worker 
473*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
474*35ffd701SAndroid Build Coastguard Worker ///  Initializes the Gfx PAT tables for AdvCtx and Gfx MMIO/Private PAT
475*35ffd701SAndroid Build Coastguard Worker ///    PAT0 = WB_COHERENT or UC depending on WaGttPat0WB
476*35ffd701SAndroid Build Coastguard Worker ///    PAT1 = UC or WB_COHERENT depending on WaGttPat0WB
477*35ffd701SAndroid Build Coastguard Worker ///    PAT2 = WB_MOCSLESS
478*35ffd701SAndroid Build Coastguard Worker ///    PAT3 = WB
479*35ffd701SAndroid Build Coastguard Worker ///    PAT4 = WT
480*35ffd701SAndroid Build Coastguard Worker ///    PAT5 = WC
481*35ffd701SAndroid Build Coastguard Worker ///    PAT6 = WC
482*35ffd701SAndroid Build Coastguard Worker ///    PAT7 = WC
483*35ffd701SAndroid Build Coastguard Worker ///  HLD says to set to PAT0/1 to WC, but since we don't have a WC in GPU,
484*35ffd701SAndroid Build Coastguard Worker ///  WC option is same as UC. Hence setting PAT0 or PAT1 to UC.
485*35ffd701SAndroid Build Coastguard Worker ///  Unused PAT's (5,6,7) are set to WC.
486*35ffd701SAndroid Build Coastguard Worker ///
487*35ffd701SAndroid Build Coastguard Worker /// @return        GMM_STATUS
488*35ffd701SAndroid Build Coastguard Worker /////////////////////////////////////////////////////////////////////////////////////
SetupPAT()489*35ffd701SAndroid Build Coastguard Worker GMM_STATUS GmmLib::GmmGen12CachePolicy::SetupPAT()
490*35ffd701SAndroid Build Coastguard Worker {
491*35ffd701SAndroid Build Coastguard Worker     GMM_STATUS Status = GMM_SUCCESS;
492*35ffd701SAndroid Build Coastguard Worker #if(defined(__GMM_KMD__))
493*35ffd701SAndroid Build Coastguard Worker     uint32_t i = 0;
494*35ffd701SAndroid Build Coastguard Worker 
495*35ffd701SAndroid Build Coastguard Worker     GMM_GFX_MEMORY_TYPE GfxMemType                 = GMM_GFX_UC_WITH_FENCE;
496*35ffd701SAndroid Build Coastguard Worker     int32_t *           pPrivatePATTableMemoryType = NULL;
497*35ffd701SAndroid Build Coastguard Worker     pPrivatePATTableMemoryType                     = pGmmLibContext->GetPrivatePATTableMemoryType();
498*35ffd701SAndroid Build Coastguard Worker 
499*35ffd701SAndroid Build Coastguard Worker     __GMM_ASSERT(pGmmLibContext->GetSkuTable().FtrIA32eGfxPTEs);
500*35ffd701SAndroid Build Coastguard Worker 
501*35ffd701SAndroid Build Coastguard Worker     for(i = 0; i < GMM_NUM_GFX_PAT_TYPES; i++)
502*35ffd701SAndroid Build Coastguard Worker     {
503*35ffd701SAndroid Build Coastguard Worker         pPrivatePATTableMemoryType[i] = -1;
504*35ffd701SAndroid Build Coastguard Worker     }
505*35ffd701SAndroid Build Coastguard Worker 
506*35ffd701SAndroid Build Coastguard Worker     // Set values for GmmGlobalInfo PrivatePATTable
507*35ffd701SAndroid Build Coastguard Worker     for(i = 0; i < NumPATRegisters; i++)
508*35ffd701SAndroid Build Coastguard Worker     {
509*35ffd701SAndroid Build Coastguard Worker         GMM_PRIVATE_PAT PAT = {0};
510*35ffd701SAndroid Build Coastguard Worker 
511*35ffd701SAndroid Build Coastguard Worker         switch(i)
512*35ffd701SAndroid Build Coastguard Worker         {
513*35ffd701SAndroid Build Coastguard Worker             case PAT0:
514*35ffd701SAndroid Build Coastguard Worker                 if(pGmmLibContext->GetWaTable().WaGttPat0)
515*35ffd701SAndroid Build Coastguard Worker                 {
516*35ffd701SAndroid Build Coastguard Worker                     if(pGmmLibContext->GetWaTable().WaGttPat0WB)
517*35ffd701SAndroid Build Coastguard Worker                     {
518*35ffd701SAndroid Build Coastguard Worker                         GfxMemType                                          = GMM_GFX_WB;
519*35ffd701SAndroid Build Coastguard Worker                         pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
520*35ffd701SAndroid Build Coastguard Worker                     }
521*35ffd701SAndroid Build Coastguard Worker                     else
522*35ffd701SAndroid Build Coastguard Worker                     {
523*35ffd701SAndroid Build Coastguard Worker                         GfxMemType                                 = GMM_GFX_UC_WITH_FENCE;
524*35ffd701SAndroid Build Coastguard Worker                         pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT0;
525*35ffd701SAndroid Build Coastguard Worker                     }
526*35ffd701SAndroid Build Coastguard Worker                 }
527*35ffd701SAndroid Build Coastguard Worker                 else // if GTT is not tied to PAT0 then WaGttPat0WB is NA
528*35ffd701SAndroid Build Coastguard Worker                 {
529*35ffd701SAndroid Build Coastguard Worker                     GfxMemType                                          = GMM_GFX_WB;
530*35ffd701SAndroid Build Coastguard Worker                     pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT0;
531*35ffd701SAndroid Build Coastguard Worker                 }
532*35ffd701SAndroid Build Coastguard Worker                 break;
533*35ffd701SAndroid Build Coastguard Worker 
534*35ffd701SAndroid Build Coastguard Worker             case PAT1:
535*35ffd701SAndroid Build Coastguard Worker                 if(pGmmLibContext->GetWaTable().WaGttPat0 && !pGmmLibContext->GetWaTable().WaGttPat0WB)
536*35ffd701SAndroid Build Coastguard Worker                 {
537*35ffd701SAndroid Build Coastguard Worker                     GfxMemType                                          = GMM_GFX_WB;
538*35ffd701SAndroid Build Coastguard Worker                     pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_COHERENT] = PAT1;
539*35ffd701SAndroid Build Coastguard Worker                 }
540*35ffd701SAndroid Build Coastguard Worker                 else
541*35ffd701SAndroid Build Coastguard Worker                 {
542*35ffd701SAndroid Build Coastguard Worker                     GfxMemType                                 = GMM_GFX_UC_WITH_FENCE;
543*35ffd701SAndroid Build Coastguard Worker                     pPrivatePATTableMemoryType[GMM_GFX_PAT_UC] = PAT1;
544*35ffd701SAndroid Build Coastguard Worker                 }
545*35ffd701SAndroid Build Coastguard Worker                 break;
546*35ffd701SAndroid Build Coastguard Worker 
547*35ffd701SAndroid Build Coastguard Worker             case PAT2:
548*35ffd701SAndroid Build Coastguard Worker                 // This PAT idx shall be used for MOCS'Less resources like Page Tables
549*35ffd701SAndroid Build Coastguard Worker                 // Page Tables have TC hardcoded to eLLC+LLC in Adv Ctxt. Hence making this to have same in Leg Ctxt.
550*35ffd701SAndroid Build Coastguard Worker                 // For BDW-H, due to Perf issue, TC has to be eLLC only for Page Tables when eDRAM is present.
551*35ffd701SAndroid Build Coastguard Worker                 GfxMemType                                          = GMM_GFX_WB;
552*35ffd701SAndroid Build Coastguard Worker                 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB_MOCSLESS] = PAT2;
553*35ffd701SAndroid Build Coastguard Worker                 break;
554*35ffd701SAndroid Build Coastguard Worker 
555*35ffd701SAndroid Build Coastguard Worker             case PAT3:
556*35ffd701SAndroid Build Coastguard Worker                 GfxMemType                                 = GMM_GFX_WB;
557*35ffd701SAndroid Build Coastguard Worker                 pPrivatePATTableMemoryType[GMM_GFX_PAT_WB] = PAT3;
558*35ffd701SAndroid Build Coastguard Worker                 break;
559*35ffd701SAndroid Build Coastguard Worker 
560*35ffd701SAndroid Build Coastguard Worker             case PAT4:
561*35ffd701SAndroid Build Coastguard Worker                 GfxMemType                                 = GMM_GFX_WT;
562*35ffd701SAndroid Build Coastguard Worker                 pPrivatePATTableMemoryType[GMM_GFX_PAT_WT] = PAT4;
563*35ffd701SAndroid Build Coastguard Worker                 break;
564*35ffd701SAndroid Build Coastguard Worker 
565*35ffd701SAndroid Build Coastguard Worker             case PAT5:
566*35ffd701SAndroid Build Coastguard Worker             case PAT6:
567*35ffd701SAndroid Build Coastguard Worker             case PAT7:
568*35ffd701SAndroid Build Coastguard Worker                 GfxMemType                                 = GMM_GFX_WC;
569*35ffd701SAndroid Build Coastguard Worker                 pPrivatePATTableMemoryType[GMM_GFX_PAT_WC] = PAT5;
570*35ffd701SAndroid Build Coastguard Worker                 break;
571*35ffd701SAndroid Build Coastguard Worker 
572*35ffd701SAndroid Build Coastguard Worker             default:
573*35ffd701SAndroid Build Coastguard Worker                 __GMM_ASSERT(0);
574*35ffd701SAndroid Build Coastguard Worker                 Status = GMM_ERROR;
575*35ffd701SAndroid Build Coastguard Worker         }
576*35ffd701SAndroid Build Coastguard Worker 
577*35ffd701SAndroid Build Coastguard Worker         PAT.Gen12.MemoryType = GfxMemType;
578*35ffd701SAndroid Build Coastguard Worker 
579*35ffd701SAndroid Build Coastguard Worker         SetPrivatePATEntry(i, PAT);
580*35ffd701SAndroid Build Coastguard Worker     }
581*35ffd701SAndroid Build Coastguard Worker 
582*35ffd701SAndroid Build Coastguard Worker #else
583*35ffd701SAndroid Build Coastguard Worker     Status = GMM_ERROR;
584*35ffd701SAndroid Build Coastguard Worker #endif
585*35ffd701SAndroid Build Coastguard Worker     return Status;
586*35ffd701SAndroid Build Coastguard Worker }
587*35ffd701SAndroid Build Coastguard Worker 
588*35ffd701SAndroid Build Coastguard Worker //=============================================================================
589*35ffd701SAndroid Build Coastguard Worker //
590*35ffd701SAndroid Build Coastguard Worker // Function: SetUpMOCSTable
591*35ffd701SAndroid Build Coastguard Worker //
592*35ffd701SAndroid Build Coastguard Worker // Desc:
593*35ffd701SAndroid Build Coastguard Worker //
594*35ffd701SAndroid Build Coastguard Worker // Parameters:
595*35ffd701SAndroid Build Coastguard Worker //
596*35ffd701SAndroid Build Coastguard Worker // Return: GMM_STATUS
597*35ffd701SAndroid Build Coastguard Worker //
598*35ffd701SAndroid Build Coastguard Worker //-----------------------------------------------------------------------------
SetUpMOCSTable()599*35ffd701SAndroid Build Coastguard Worker void GmmLib::GmmGen12CachePolicy::SetUpMOCSTable()
600*35ffd701SAndroid Build Coastguard Worker {
601*35ffd701SAndroid Build Coastguard Worker     GMM_CACHE_POLICY_TBL_ELEMENT *pCachePolicyTlbElement = &(pGmmLibContext->GetCachePolicyTlbElement()[0]);
602*35ffd701SAndroid Build Coastguard Worker 
603*35ffd701SAndroid Build Coastguard Worker #define GMM_DEFINE_MOCS(Index, L3_ESC, L3_SCC, L3_CC, LeCC_CC, LeCC_TC, LeCC_LRUM, LeCC_AOM, LeCC_ESC, LeCC_SCC, LeCC_PFM, LeCC_SCF, LeCC_CoS, LeCC_SelfSnoop, _HDCL1) \
604*35ffd701SAndroid Build Coastguard Worker     {                                                                                                                                                                  \
605*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].L3.ESC            = L3_ESC;                                                                                                      \
606*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].L3.SCC            = L3_SCC;                                                                                                      \
607*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].L3.Cacheability   = L3_CC;                                                                                                       \
608*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.Cacheability = LeCC_CC;                                                                                                     \
609*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.TargetCache  = LeCC_TC;                                                                                                     \
610*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.LRUM         = LeCC_LRUM;                                                                                                   \
611*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.AOM          = LeCC_AOM;                                                                                                    \
612*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.ESC          = LeCC_ESC;                                                                                                    \
613*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.SCC          = LeCC_SCC;                                                                                                    \
614*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.PFM          = LeCC_PFM;                                                                                                    \
615*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.SCF          = LeCC_SCF;                                                                                                    \
616*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.CoS          = LeCC_CoS;                                                                                                    \
617*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].LeCC.SelfSnoop    = LeCC_SelfSnoop;                                                                                              \
618*35ffd701SAndroid Build Coastguard Worker         pCachePolicyTlbElement[Index].HDCL1             = _HDCL1;                                                                                                      \
619*35ffd701SAndroid Build Coastguard Worker     }
620*35ffd701SAndroid Build Coastguard Worker 
621*35ffd701SAndroid Build Coastguard Worker     // clang-format off
622*35ffd701SAndroid Build Coastguard Worker 
623*35ffd701SAndroid Build Coastguard Worker    //Default MOCS Table
624*35ffd701SAndroid Build Coastguard Worker     for(int index = 0; index < GMM_MAX_NUMBER_MOCS_INDEXES; index++)
625*35ffd701SAndroid Build Coastguard Worker     {      //           Index     ES     SCC     L3CC    LeCC    TC      LRUM    DAoM    ERSC    SCC     PFM    SCF     CoS     SSE   HDCL1
626*35ffd701SAndroid Build Coastguard Worker         GMM_DEFINE_MOCS( index  , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
627*35ffd701SAndroid Build Coastguard Worker     }
628*35ffd701SAndroid Build Coastguard Worker 
629*35ffd701SAndroid Build Coastguard Worker     // Fixed MOCS Table
630*35ffd701SAndroid Build Coastguard Worker     //              Index     ESC	  SCC	  L3CC    LeCC    TC      LRUM    DAoM	  ERSC	  SCC	  PFM	  SCF     CoS     SSE   HDCL1
631*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 1      , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
632*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 2      , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
633*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 3      , 0     , 0     , 1     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
634*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 4      , 0     , 0     , 3     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
635*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 5      , 0     , 0     , 1     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
636*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 6      , 0     , 0     , 1     , 3     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
637*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 7      , 0     , 0     , 3     , 3     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
638*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 8      , 0     , 0     , 1     , 3     , 1     , 2     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
639*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 9      , 0     , 0     , 3     , 3     , 1     , 2     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
640*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 10     , 0     , 0     , 1     , 3     , 1     , 3     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
641*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 11     , 0     , 0     , 3     , 3     , 1     , 3     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
642*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 12     , 0     , 0     , 1     , 3     , 1     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
643*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 13     , 0     , 0     , 3     , 3     , 1     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
644*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 14     , 0     , 0     , 1     , 3     , 1     , 2     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
645*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 15     , 0     , 0     , 3     , 3     , 1     , 2     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
646*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 16     , 0     , 0     , 1     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 1     , 0     , 0     , 0 )
647*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 17     , 0     , 0     , 3     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 1     , 0     , 0     , 0 )
648*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 18     , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 3     , 0 )
649*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 19     , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 0     , 7     , 0     , 0     , 0     , 0     , 0 )
650*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 20     , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 0     , 3     , 0     , 0     , 0     , 0     , 0 )
651*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 21     , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 0     , 1     , 0     , 0     , 0     , 0     , 0 )
652*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 22     , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 1     , 3     , 0     , 0     , 0     , 0     , 0 )
653*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 23     , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 1     , 7     , 0     , 0     , 0     , 0     , 0 )
654*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 48     , 0     , 0     , 3     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 1 )
655*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 49     , 0     , 0     , 3     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 1 )
656*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 50     , 0     , 0     , 1     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 1 )
657*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 51     , 0     , 0     , 1     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 1 )
658*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 60     , 0     , 0     , 1     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
659*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 61     , 0     , 0     , 3     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 1     , 0     , 0     , 0 )
660*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 62     , 0     , 0     , 1     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
661*35ffd701SAndroid Build Coastguard Worker     GMM_DEFINE_MOCS( 63     , 0     , 0     , 1     , 3     , 1     , 3     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
662*35ffd701SAndroid Build Coastguard Worker 
663*35ffd701SAndroid Build Coastguard Worker 
664*35ffd701SAndroid Build Coastguard Worker     if(!pGmmLibContext->GetSkuTable().FtrLLCBypass ||
665*35ffd701SAndroid Build Coastguard Worker         GFX_GET_CURRENT_PRODUCT(pGmmLibContext->GetPlatformInfo().Platform) == IGFX_ROCKETLAKE)
666*35ffd701SAndroid Build Coastguard Worker     {
667*35ffd701SAndroid Build Coastguard Worker         GMM_DEFINE_MOCS( 16     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
668*35ffd701SAndroid Build Coastguard Worker         GMM_DEFINE_MOCS( 17     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
669*35ffd701SAndroid Build Coastguard Worker         GMM_DEFINE_MOCS( 61     , 0     , 0     , 3     , 1     , 1     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0     , 0 )
670*35ffd701SAndroid Build Coastguard Worker     }
671*35ffd701SAndroid Build Coastguard Worker     // clang-format on
672*35ffd701SAndroid Build Coastguard Worker 
673*35ffd701SAndroid Build Coastguard Worker     CurrentMaxMocsIndex        = 23;
674*35ffd701SAndroid Build Coastguard Worker     CurrentMaxL1HdcMocsIndex   = 51;
675*35ffd701SAndroid Build Coastguard Worker     CurrentMaxSpecialMocsIndex = 63;
676*35ffd701SAndroid Build Coastguard Worker 
677*35ffd701SAndroid Build Coastguard Worker #undef GMM_DEFINE_MOCS
678*35ffd701SAndroid Build Coastguard Worker }
679