1 /*==============================================================================
2 Copyright(c) 2019 Intel Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a
5 copy of this software and associated documentation files(the "Software"),
6 to deal in the Software without restriction, including without limitation
7 the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and / or sell copies of the Software, and to permit persons to whom the
9 Software is furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included
12 in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 OTHER DEALINGS IN THE SOFTWARE.
21
22 Description: AUX-Table management functions
23 (contains functions to assign memory to
24 AUX-Tables with valid entries,
25 and update their entries on request)
26
27 ============================================================================*/
28
29 #include "Internal/Common/GmmLibInc.h"
30 #include "../TranslationTable/GmmUmdTranslationTable.h"
31
32 #if !defined(__GMM_KMD__)
33
34 //=============================================================================
35 //
36 // Function: MapNullCCS
37 //
38 // Desc: Maps given resource, with dummy null-ccs chain, on Aux Table
39 //
40 // Caller: UpdateAuxTable (map op for null-tiles)
41 //
42 // Parameters:
43 // UmdContext: Caller-thread specific info (regarding BB for TR-Aux udpate, cmdQ to use etc)
44 // BaseAdr: Start adr of main surface
45 // Size: Main-surface size in bytes
46 // PartialL1e: Aux-metadata other than AuxVA
47 // DoNotWait: 1 for CPU update, 0 for async(Gpu) update
48 //-----------------------------------------------------------------------------
MapNullCCS(GMM_UMD_SYNCCONTEXT * UmdContext,GMM_GFX_ADDRESS BaseAdr,GMM_GFX_SIZE_T Size,uint64_t PartialL1e,uint8_t DoNotWait)49 GMM_STATUS GmmLib::AuxTable::MapNullCCS(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint64_t PartialL1e, uint8_t DoNotWait)
50 {
51 GMM_STATUS Status = GMM_SUCCESS;
52 GMM_GFX_SIZE_T L1TableSize = ((GMM_GFX_SIZE_T)GMM_L1_SIZE(AUXTT, GetGmmLibContext())) * (WA16K(GetGmmLibContext()) ? GMM_KBYTE(16) : GMM_KBYTE(64)); // TGL and above : L1TableSize = 256x64K OR 16x1M
53 GMM_GFX_ADDRESS Addr = 0;
54 GMM_GFX_ADDRESS L3GfxAddress = 0;
55 GMM_CLIENT ClientType;
56
57 GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
58
59 EnterCriticalSection(&TTLock);
60
61 DoNotWait |= (!UmdContext || !UmdContext->pCommandQueueHandle);
62
63 if(TTL3.L3Handle)
64 {
65 L3GfxAddress = TTL3.GfxAddress;
66 }
67 else
68 {
69 LeaveCriticalSection(&TTLock);
70 return GMM_ERROR;
71 }
72
73 if(!DoNotWait)
74 {
75 PageTableMgr->TTCb.pfPrologTranslationTable(
76 UmdContext->pCommandQueueHandle);
77 }
78
79 // For each L1 table
80 for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); // Start at begining of L1 table
81 Addr < BaseAdr + Size;
82 Addr += L1TableSize) // Increment by 1 L1 table
83 {
84 GMM_GFX_ADDRESS L1GfxAddress, L2GfxAddress;
85 GMM_GFX_ADDRESS L1CPUAddress, L2CPUAddress;
86 GMM_GFX_ADDRESS StartAddress = 0;
87 GMM_GFX_ADDRESS EndAddress = 0;
88 GMM_GFX_ADDRESS TileAddr = 0;
89 GMM_GFX_SIZE_T L2eIdx = 0;
90
91 StartAddress = Addr < BaseAdr ? BaseAdr : Addr;
92 EndAddress = Addr + L1TableSize;
93 if(EndAddress > BaseAdr + Size)
94 {
95 EndAddress = BaseAdr + Size;
96 }
97
98 GetL1L2TableAddr(StartAddress,
99 &L1GfxAddress,
100 &L2GfxAddress);
101
102 // If tables are not there, then they are already invalidated as part of
103 // AUX-TT initialization or other APIs.
104 if(L2GfxAddress == GMM_NO_TABLE ||
105 L1GfxAddress == GMM_NO_TABLE)
106 {
107 //Clear Valid-bit for L3Entry or L2Entry
108 uint64_t Data = 0;
109 GMM_GFX_ADDRESS TableGfxAddress = (L2GfxAddress == GMM_NO_TABLE) ? L3GfxAddress : L2GfxAddress;
110 GMM_GFX_ADDRESS TableCPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? TTL3.CPUAddress : pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].GetCPUAddress();
111 uint32_t TableEntryIdx = (L2GfxAddress == GMM_NO_TABLE) ? static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(AUXTT, StartAddress));
112 L2CPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? 0 : TableCPUAddress;
113
114 if(!NullL1Table || !NullL2Table)
115 {
116 AllocateDummyTables(&NullL2Table, &NullL1Table);
117 if(!NullL1Table || !NullL2Table)
118 {
119 //report error
120 LeaveCriticalSection(&TTLock);
121 return GMM_OUT_OF_MEMORY;
122 }
123 else
124 {
125 //Initialize dummy table entries (one-time)
126 GMM_GFX_ADDRESS TableAddr = NullL2Table->GetCPUAddress();
127 GMM_AUXTTL2e L2e = {0};
128 L2e.Valid = 1;
129 GMM_TO_AUX_L2e_L1GFXADDR_2((NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()), L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext()))) // populate L2e.L1GfxAddr
130 for(int i = 0; i < GMM_AUX_L2_SIZE; i++)
131 {
132 //initialize L2e ie clear Valid bit for all entries
133 ((GMM_AUXTTL2e *)TableAddr)[i].Value = L2e.Value;
134 }
135
136 TableAddr = NullL1Table->GetCPUAddress();
137
138 GMM_AUXTTL1e L1e = {0};
139 L1e.Valid = 1;
140 if(!WA64K(GetGmmLibContext()))
141 {
142 L1e.GfxAddress = (NullCCSTile >> 12); /*********** 4kb-aligned CCS adr *****/
143 }
144 else
145 {
146 L1e.Reserved4 = (NullCCSTile >> 8); /*********** 4 lsbs of 256B-aligned CCS adr *****/
147 L1e.GfxAddress = (NullCCSTile >> 12); /*********** 4kb-aligned CCS adr *****/
148 }
149
150 for(int i = 0; i < GMM_AUX_L1_SIZE(GetGmmLibContext()); i++)
151 {
152 //initialize L1e with null ccs tile
153 ((GMM_AUXTTL1e *)TableAddr)[i].Value = L1e.Value;
154 }
155 }
156 }
157
158 if(L2GfxAddress == GMM_NO_TABLE)
159 {
160 GMM_AUXTTL3e L3e = {0};
161 L3e.Valid = 1;
162 L3e.L2GfxAddr = (NullL2Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL2Table->GetNodeIdx()) >> 15;
163 Data = L3e.Value;
164 }
165 else
166 {
167 GMM_AUXTTL2e L2e = {0};
168 L2e.Valid = 1;
169 GMM_TO_AUX_L2e_L1GFXADDR_2((NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()), L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
170 Data = L2e.Value;
171 }
172
173 if(DoNotWait)
174 {
175 //Sync update on CPU
176 ((GMM_AUXTTL2e *)TableCPUAddress)[TableEntryIdx].Value = Data;
177 }
178 else
179 {
180 if(L2GfxAddress != GMM_NO_TABLE)
181 {
182 pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].UpdatePoolFence(UmdContext, false);
183 }
184 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
185 TableGfxAddress + TableEntryIdx * GMM_AUX_L2e_SIZE,
186 Data);
187 }
188 continue;
189 }
190 else
191 {
192 uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress));
193 L2CPUAddress = pTTL2[L3eIdx].GetCPUAddress();
194
195 L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAddress);
196 if(DoNotWait)
197 {
198 //Sync update on CPU
199 ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1; //set Valid bit
200 ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2GfxAddress >> 15;
201
202 ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid = 1; //set Valid bit
203 GMM_TO_AUX_L2e_L1GFXADDR_2(L1GfxAddress, ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx], (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
204 }
205 else
206 {
207 GMM_AUXTTL3e L3e = {0};
208 L3e.Valid = 1;
209 L3e.L2GfxAddr = L2GfxAddress >> 15;
210 PageTableMgr->TTCb.pfWriteL2L3Entry(
211 UmdContext->pCommandQueueHandle,
212 L3GfxAddress + (L3eIdx * GMM_AUX_L3e_SIZE),
213 L3e.Value);
214
215 pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
216
217 GMM_AUXTTL2e L2e = {0};
218 L2e.Valid = 1;
219 GMM_TO_AUX_L2e_L1GFXADDR_2(L1GfxAddress, L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
220 PageTableMgr->TTCb.pfWriteL2L3Entry(
221 UmdContext->pCommandQueueHandle,
222 L2GfxAddress + (L2eIdx * GMM_AUX_L2e_SIZE),
223 L2e.Value);
224 }
225 }
226
227 // For each 64KB or 16KB of main surface (entry) in L1 table
228 for(TileAddr = StartAddress;
229 TileAddr < EndAddress;
230 TileAddr += (WA16K(GetGmmLibContext()) ? GMM_KBYTE(16) : WA64K(GetGmmLibContext()) ? GMM_KBYTE(64) : GMM_MBYTE(1)))
231 {
232 uint64_t Data = PartialL1e | NullCCSTile | __BIT(0);
233 GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAddr, GetGmmLibContext());
234 GmmLib::LastLevelTable *pL1Tbl = NULL;
235
236 pL1Tbl = pTTL2[GMM_AUX_L3_ENTRY_IDX(TileAddr)].GetL1Table(L2eIdx, NULL);
237 L1CPUAddress = pL1Tbl->GetCPUAddress();
238 if(DoNotWait)
239 {
240 //Sync update on CPU
241 ((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].Value = Data;
242 GMM_DPF(GFXDBG_NORMAL, "##### Null-Map | Table Entry: TileAddress[0x%llX] L2eIdx[%d] :: L1eIdx[%d] L1Addr[0x%llX] L1Value[00x%llX]\n", TileAddr, L2eIdx, L1eIdx, &((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx], Data);
243 }
244 else
245 {
246 pL1Tbl->UpdatePoolFence(UmdContext, false);
247
248 /* PageTableMgr->TTCb.pfWriteL1Entries(
249 UmdContext->pCommandQueueHandle,
250 2,
251 L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
252 (uint32_t*)(&Data));*/ //**********REQUIRE UMD CHANGE TO UPDATE 64-bit ENTRY - both DWORDs must be updated atomically*******/
253 PageTableMgr->TTCb.pfWriteL2L3Entry(
254 UmdContext->pCommandQueueHandle,
255 L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
256 Data);
257 }
258
259 if(pL1Tbl->TrackTableUsage(AUXTT, true, TileAddr, true, GetGmmLibContext()))
260 { // L1 Table is not being used anymore
261 GMM_AUXTTL2e L2e = {0};
262 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
263 GmmLib::LastLevelTable * pL1Tbl = NULL, *Prev = NULL;
264
265 pL1Tbl = pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].GetL1Table(L2eIdx, &Prev);
266 // Map L2-entry to Null-L1Table
267 L2e.Valid = 1;
268 GMM_TO_AUX_L2e_L1GFXADDR_2((NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()), L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext()))) // populate L2e.L1GfxAddress/Le2.Reserved2
269 if(DoNotWait)
270 {
271 //Sync update on CPU
272 ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Value = L2e.Value;
273 }
274 else
275 {
276 pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].UpdatePoolFence(UmdContext, false);
277 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
278 L2GfxAddress + L2eIdx * GMM_AUX_L2e_SIZE,
279 L2e.Value);
280 }
281 //Update usage for PoolNode assigned to L1Table, and free L1Tbl
282 if(pL1Tbl)
283 {
284 PoolElem = pL1Tbl->GetPool();
285 if(PoolElem)
286 {
287 if(pL1Tbl->GetBBInfo().BBQueueHandle)
288 {
289 PoolElem->GetNodeBBInfoAtIndex(pL1Tbl->GetNodeIdx()) = pL1Tbl->GetBBInfo();
290 }
291 DEASSIGN_POOLNODE(PageTableMgr, UmdContext, PoolElem, pL1Tbl->GetNodeIdx(), AUX_L1TABLE_SIZE_IN_POOLNODES_2(GetGmmLibContext()))
292 }
293 pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].DeleteFromList(pL1Tbl, Prev);
294 }
295
296 // The L1 table is unused -- meaning everything else in this table is
297 // already invalid. So, break early.
298 break;
299 }
300 }
301 }
302
303 if(!DoNotWait)
304 {
305 PageTableMgr->TTCb.pfEpilogTranslationTable(
306 UmdContext->pCommandQueueHandle,
307 1); // ForceFlush
308 }
309 LeaveCriticalSection(&TTLock);
310
311 return Status;
312 }
313
314 //=============================================================================
315 //
316 // Function: InvalidateTable (InvalidateMappings)
317 //
318 // Desc: Unmaps given resource from Aux Table; and marks affected entries as invalid
319 //
320 // Caller: UpdateAuxTable (unmap op)
321 //
322 // Parameters:
323 // UmdContext: Caller-thread specific info (regarding BB for Aux udpate, cmdQ to use etc)
324 // BaseAdr: Start adr of main surface
325 // Size: Main-surface size in bytes? (or take GmmResInfo?)
326 // DoNotWait: 1 for CPU update, 0 for async(Gpu) update
327 //-----------------------------------------------------------------------------
InvalidateTable(GMM_UMD_SYNCCONTEXT * UmdContext,GMM_GFX_ADDRESS BaseAdr,GMM_GFX_SIZE_T Size,uint8_t DoNotWait)328 GMM_STATUS GmmLib::AuxTable::InvalidateTable(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint8_t DoNotWait)
329 {
330 GMM_STATUS Status = GMM_SUCCESS;
331 GMM_GFX_SIZE_T L1TableSize = ((GMM_GFX_SIZE_T)GMM_L1_SIZE(AUXTT, GetGmmLibContext())) * (WA16K(GetGmmLibContext()) ? GMM_KBYTE(16) : GMM_KBYTE(64)); //Each AuxTable entry maps 16K main-surface
332 GMM_GFX_ADDRESS Addr = 0;
333 GMM_GFX_ADDRESS L3GfxAddress = 0;
334 uint8_t isTRVA = 0;
335
336 GMM_CLIENT ClientType;
337
338 GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
339
340 //NullCCSTile isn't initialized, disable TRVA path
341 isTRVA = (NullCCSTile ? isTRVA : 0);
342
343 EnterCriticalSection(&TTLock);
344
345 DoNotWait |= (!UmdContext || !UmdContext->pCommandQueueHandle);
346
347 if(TTL3.L3Handle)
348 {
349 L3GfxAddress = TTL3.GfxAddress;
350 }
351 else
352 {
353 LeaveCriticalSection(&TTLock);
354 return GMM_ERROR;
355 }
356
357 if(!DoNotWait)
358 {
359 PageTableMgr->TTCb.pfPrologTranslationTable(
360 UmdContext->pCommandQueueHandle);
361 }
362
363 // For each L1 table
364 for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); // Start at begining of L1 table
365 Addr < BaseAdr + Size;
366 Addr += L1TableSize) // Increment by 1 L1 table
367 {
368 GMM_GFX_ADDRESS L1GfxAddress, L2GfxAddress;
369 GMM_GFX_ADDRESS L1CPUAddress, L2CPUAddress;
370 GMM_GFX_ADDRESS StartAddress = 0;
371 GMM_GFX_ADDRESS EndAddress = 0;
372 GMM_GFX_ADDRESS TileAddr = 0;
373 GMM_GFX_SIZE_T L2eIdx = 0;
374
375 StartAddress = Addr < BaseAdr ? BaseAdr : Addr;
376 EndAddress = Addr + L1TableSize;
377 if(EndAddress > BaseAdr + Size)
378 {
379 EndAddress = BaseAdr + Size;
380 }
381
382 GetL1L2TableAddr(StartAddress,
383 &L1GfxAddress,
384 &L2GfxAddress);
385
386 // If tables are not there, then they are already invalidated as part of
387 // AUX-TT initialization or other APIs.
388 if(L2GfxAddress == GMM_NO_TABLE ||
389 L1GfxAddress == GMM_NO_TABLE)
390 {
391 //Clear Valid-bit for L3Entry or L2Entry
392 GMM_AUXTTL2e L2e = {0}; //AUXTT L3e is identical to L2e, reuse.
393 GMM_GFX_ADDRESS TableGfxAddress = (L2GfxAddress == GMM_NO_TABLE) ? L3GfxAddress : L2GfxAddress;
394 GMM_GFX_ADDRESS TableCPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? TTL3.CPUAddress : pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].GetCPUAddress();
395 uint32_t TableEntryIdx = (L2GfxAddress == GMM_NO_TABLE) ? static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(AUXTT, StartAddress));
396 L2CPUAddress = (L2GfxAddress == GMM_NO_TABLE) ? 0 : TableCPUAddress;
397
398 if(isTRVA && NullL2Table && NullL1Table)
399 {
400 //invalidate if request spans entire stretch ie TileAdr aligns L1TableSize*GMM_L2_SIZE
401 uint64_t Data = 0;
402 if(L2GfxAddress == GMM_NO_TABLE)
403 {
404 GMM_AUXTTL3e L3e = {0};
405 L3e.Valid = 1;
406 L3e.L2GfxAddr = (NullL2Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL2Table->GetNodeIdx()) >> 15;
407 Data = L3e.Value;
408 }
409 else
410 {
411 GMM_AUXTTL2e L2e = {0};
412 L2e.Valid = 1;
413 GMM_TO_AUX_L2e_L1GFXADDR_2((NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()), L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
414 Data = L2e.Value;
415 }
416 L2e.Value = Data;
417 }
418 else
419 {
420 L2e.Valid = 0;
421 }
422 if(DoNotWait)
423 {
424 //Sync update on CPU
425 ((GMM_AUXTTL2e *)TableCPUAddress)[TableEntryIdx].Value = L2e.Value;
426 }
427 else
428 {
429 if(L2GfxAddress != GMM_NO_TABLE)
430 {
431 pTTL2[GMM_L3_ENTRY_IDX(AUXTT, StartAddress)].UpdatePoolFence(UmdContext, false);
432 }
433 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
434 TableGfxAddress + TableEntryIdx * GMM_AUX_L2e_SIZE,
435 L2e.Value);
436 }
437 continue;
438 }
439 else
440 {
441 uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(AUXTT, StartAddress));
442 L2CPUAddress = pTTL2[L3eIdx].GetCPUAddress();
443
444 L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAddress);
445 if(DoNotWait)
446 {
447 //Sync update on CPU
448 ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1; //set Valid bit
449 ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2GfxAddress >> 15;
450
451 ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid = 1; //set Valid bit
452 GMM_TO_AUX_L2e_L1GFXADDR_2(L1GfxAddress, ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx], (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
453 }
454 else
455 {
456 GMM_AUXTTL3e L3e = {0};
457 L3e.Valid = 1;
458 L3e.L2GfxAddr = L2GfxAddress >> 15;
459 PageTableMgr->TTCb.pfWriteL2L3Entry(
460 UmdContext->pCommandQueueHandle,
461 L3GfxAddress + (L3eIdx * GMM_AUX_L3e_SIZE),
462 L3e.Value);
463
464 pTTL2[L3eIdx].UpdatePoolFence(UmdContext, false);
465
466 GMM_AUXTTL2e L2e = {0};
467 L2e.Valid = 1;
468 GMM_TO_AUX_L2e_L1GFXADDR_2(L1GfxAddress, L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
469 PageTableMgr->TTCb.pfWriteL2L3Entry(
470 UmdContext->pCommandQueueHandle,
471 L2GfxAddress + (L2eIdx * GMM_AUX_L2e_SIZE),
472 L2e.Value);
473 }
474 }
475
476 // For each 64KB or 16KB or 1MB of main surface (entry) in L1 table
477 for(TileAddr = StartAddress;
478 TileAddr < EndAddress;
479 TileAddr += (WA16K(GetGmmLibContext()) ? GMM_KBYTE(16) : WA64K(GetGmmLibContext()) ? GMM_KBYTE(64) : GMM_MBYTE(1)))
480 {
481 //Invalidation of requested range irrespective of TRVA
482 uint64_t Data = GMM_INVALID_AUX_ENTRY;
483 GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAddr, GetGmmLibContext());
484 GmmLib::LastLevelTable *pL1Tbl = NULL;
485
486 pL1Tbl = pTTL2[GMM_AUX_L3_ENTRY_IDX(TileAddr)].GetL1Table(L2eIdx, NULL);
487 L1CPUAddress = pL1Tbl->GetCPUAddress();
488 if(DoNotWait)
489 {
490 //Sync update on CPU
491 ((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].Value = Data;
492 GMM_DPF(GFXDBG_NORMAL, "~~UnMap | Table Entry: L2addressBase[0x%llX] :: L2Valid[%d] :: L2eidx[%d] L1addressBase[0x%llX] :: L1eidx[%d] L1Valid[%d] :: DerivedCCS[0x%llX] ", (GMM_AUXTTL2e *)L2CPUAddress, ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid, L2eIdx, GMM_L1TABLE_ADDR_FROM_AUX_L2e_L1GFXADDR(((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx], true), L1eIdx, ((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].Valid, (((GMM_AUXTTL1e *)L1CPUAddress)[L1eIdx].GfxAddress << 12));
493
494 }
495 else
496 {
497 pL1Tbl->UpdatePoolFence(UmdContext, false);
498
499 /* PageTableMgr->TTCb.pfWriteL1Entries(
500 UmdContext->pCommandQueueHandle,
501 2,
502 (uint32_t*)(&Data));*/
503 //**********REQUIRE UMD CHANGE TO UPDATE 64-bit ENTRY - both DWORDs must be updated atomically*******/
504 PageTableMgr->TTCb.pfWriteL2L3Entry(
505 UmdContext->pCommandQueueHandle,
506 L1GfxAddress + (L1eIdx * GMM_AUX_L1e_SIZE),
507 Data);
508 }
509
510 if(pL1Tbl->TrackTableUsage(AUXTT, true, TileAddr, true, GetGmmLibContext()))
511 { // L1 Table is not being used anymore
512 GMM_AUXTTL2e L2e = {0};
513 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
514 GmmLib::LastLevelTable * pL1Tbl = NULL, *Prev = NULL;
515
516 pL1Tbl = pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].GetL1Table(L2eIdx, &Prev);
517
518 if(isTRVA && NullL1Table &&
519 ((TileAddr > GFX_ALIGN_FLOOR(BaseAdr, L1TableSize) && TileAddr < GFX_ALIGN_NP2(BaseAdr, L1TableSize)) ||
520 (TileAddr > GFX_ALIGN_FLOOR(BaseAdr + Size, L1TableSize) && TileAddr < GFX_ALIGN_NP2(BaseAdr + Size, L1TableSize))))
521 {
522 //Invalidation affects entries out of requested range, null-map for TR
523 L2e.Valid = 1;
524 GMM_TO_AUX_L2e_L1GFXADDR_2((NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()), L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
525 }
526 else
527 {
528 // Clear valid bit of L2 entry
529 L2e.Valid = 0;
530 ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Valid = 0;
531 }
532 if(DoNotWait)
533 {
534 //Sync update on CPU
535 ((GMM_AUXTTL2e *)L2CPUAddress)[L2eIdx].Value = L2e.Value;
536 }
537 else
538 {
539 pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].UpdatePoolFence(UmdContext, false);
540 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
541 L2GfxAddress + L2eIdx * GMM_AUX_L2e_SIZE,
542 L2e.Value);
543 }
544 //Update usage for PoolNode assigned to L1Table, and free L1Tbl
545 if(pL1Tbl)
546 {
547 PoolElem = pL1Tbl->GetPool();
548 if(PoolElem)
549 {
550 if(pL1Tbl->GetBBInfo().BBQueueHandle)
551 {
552 PoolElem->GetNodeBBInfoAtIndex(pL1Tbl->GetNodeIdx()) = pL1Tbl->GetBBInfo();
553 }
554 DEASSIGN_POOLNODE(PageTableMgr, UmdContext, PoolElem, pL1Tbl->GetNodeIdx(), AUX_L1TABLE_SIZE_IN_POOLNODES_2(GetGmmLibContext()))
555 }
556 pTTL2[GMM_L3_ENTRY_IDX(AUXTT, TileAddr)].DeleteFromList(pL1Tbl, Prev);
557 }
558
559 // The L1 table is unused -- meaning everything else in this table is
560 // already invalid. So, break early.
561 break;
562 }
563 }
564 }
565
566 if(!DoNotWait)
567 {
568 PageTableMgr->TTCb.pfEpilogTranslationTable(
569 UmdContext->pCommandQueueHandle,
570 1); // ForceFlush
571 }
572
573 LeaveCriticalSection(&TTLock);
574
575 return Status;
576 }
577
578 //=============================================================================
579 //
580 // Function: MapValidEntry
581 //
582 // Desc: Maps given main-surface, on Aux-Table, to get the exact CCS cacheline tied to
583 // different 4x4K pages of main-surface
584 //
585 // Caller: UpdateAuxTable (map op)
586 //
587 // Parameters:
588 // UmdContext: ptr to thread-data
589 // BaseAdr: Start adr of main-surface
590 // BaseSize: main-surface Size in bytes
591 // BaseResInfo: main surface ResInfo
592 // AuxVA: Start adr of Aux-surface
593 // AuxResInfo: Aux surface ResInfo
594 // PartialData: Aux L1 partial data (ie w/o address)
595 // DoNotWait: true for CPU update, false for async(Gpu) update
596 //-----------------------------------------------------------------------------
MapValidEntry(GMM_UMD_SYNCCONTEXT * UmdContext,GMM_GFX_ADDRESS BaseAdr,GMM_GFX_SIZE_T BaseSize,GMM_RESOURCE_INFO * BaseResInfo,GMM_GFX_ADDRESS AuxVA,GMM_RESOURCE_INFO * AuxResInfo,uint64_t PartialData,uint8_t DoNotWait)597 GMM_STATUS GmmLib::AuxTable::MapValidEntry(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T BaseSize,
598 GMM_RESOURCE_INFO *BaseResInfo, GMM_GFX_ADDRESS AuxVA, GMM_RESOURCE_INFO *AuxResInfo, uint64_t PartialData, uint8_t DoNotWait)
599 {
600 GMM_STATUS Status = GMM_SUCCESS;
601 GMM_GFX_ADDRESS Addr = 0, L3TableAdr = GMM_NO_TABLE;
602 GMM_GFX_SIZE_T L1TableSize = GMM_AUX_L1_SIZE(GetGmmLibContext()) * (WA16K(GetGmmLibContext()) ? GMM_KBYTE(16) : GMM_KBYTE(64)); // L1TableSize maps to 16MB address space for TGL and above: 256x64k | 16x1MB
603 GMM_GFX_SIZE_T CCS$Adr = AuxVA;
604 uint8_t isTRVA =0 ;
605
606 GMM_CLIENT ClientType;
607
608 GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
609
610 //NullCCSTile isn't initialized, disable TRVA path
611 isTRVA = (NullCCSTile ? isTRVA : 0);
612
613 EnterCriticalSection(&TTLock);
614 if(!TTL3.L3Handle || (!DoNotWait && !UmdContext))
615 {
616 Status = GMM_ERROR;
617 }
618 else
619 {
620 L3TableAdr = TTL3.GfxAddress;
621
622 if(!DoNotWait)
623 {
624 PageTableMgr->TTCb.pfPrologTranslationTable(UmdContext->pCommandQueueHandle);
625 }
626
627 // GMM_DPF(GFXDBG_CRITICAL, "Mapping surface: GPUVA=0x%016llX Size=0x%08X Aux_GPUVA=0x%016llX\n", BaseAdr, BaseSize, AuxVA);
628 for(Addr = GFX_ALIGN_FLOOR(BaseAdr, L1TableSize); Addr < BaseAdr + BaseSize; Addr += L1TableSize)
629 {
630 GMM_GFX_ADDRESS StartAdr, EndAdr, TileAdr;
631 GMM_GFX_ADDRESS L1TableAdr = GMM_NO_TABLE, L2TableAdr = GMM_NO_TABLE;
632 GMM_GFX_ADDRESS L1TableCPUAdr = GMM_NO_TABLE, L2TableCPUAdr = GMM_NO_TABLE;
633 GMM_GFX_SIZE_T L2eIdx = 0;
634 GMM_GFX_SIZE_T L3eIdx = 0;
635 bool AllocateL1 = false, AllocateL2 = false;
636
637 EndAdr = Addr + L1TableSize;
638 EndAdr = EndAdr > BaseAdr + BaseSize ? BaseAdr + BaseSize : EndAdr;
639 StartAdr = Addr < BaseAdr ? BaseAdr : Addr;
640
641 L2eIdx = GMM_L2_ENTRY_IDX(AUXTT, StartAdr);
642 L3eIdx = GMM_L3_ENTRY_IDX(AUXTT, StartAdr);
643
644 //Allocate L2/L1 Table -- get L2 Table Adr for <StartAdr,EndAdr>
645 GetL1L2TableAddr(Addr, &L1TableAdr, &L2TableAdr);
646 if(L2TableAdr == GMM_NO_TABLE || L1TableAdr == GMM_NO_TABLE)
647 {
648 AllocateL1 = GMM_NO_TABLE == L1TableAdr;
649 AllocateL2 = GMM_NO_TABLE == L2TableAdr;
650 AllocateL1L2Table(Addr, &L1TableAdr, &L2TableAdr);
651
652 if(L2TableAdr == GMM_NO_TABLE || L1TableAdr == GMM_NO_TABLE)
653 {
654 LeaveCriticalSection(&TTLock);
655 return GMM_OUT_OF_MEMORY;
656 }
657
658 if(AllocateL2)
659 {
660 uint32_t i = 0;
661 GMM_AUXTTL2e InvalidEntry;
662 InvalidEntry.Value = 0;
663 if(isTRVA && NullL1Table)
664 {
665 InvalidEntry.Valid = 1;
666 GMM_TO_AUX_L2e_L1GFXADDR_2((NullL1Table->GetPool()->GetGfxAddress() + PAGE_SIZE * NullL1Table->GetNodeIdx()), InvalidEntry, (!WA16K(GetGmmLibContext()) && !WA64K(GetGmmLibContext())))
667 }
668
669 if(DoNotWait)
670 {
671 L2TableCPUAdr = pTTL2[L3eIdx].GetCPUAddress();
672
673 ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Value = 0;
674 ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr = L2TableAdr >> 15;
675 ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid = 1;
676 for(i = 0; i < GMM_AUX_L2_SIZE; i++)
677 {
678 //initialize L2e ie clear Valid bit for all entries
679 ((GMM_AUXTTL2e *)L2TableCPUAdr)[i].Value = InvalidEntry.Value;
680 }
681 }
682 else
683 {
684 GMM_AUXTTL3e L3e = {0};
685 L3e.Valid = 1;
686 L3e.L2GfxAddr = L2TableAdr >> 15;
687 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
688 L3TableAdr + L3eIdx * GMM_AUX_L3e_SIZE,
689 L3e.Value);
690
691 //initialize L2e ie clear valid bit for all entries
692 for(i = 0; i < GMM_AUX_L2_SIZE; i++)
693 {
694 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
695 L2TableAdr + i * GMM_AUX_L2e_SIZE,
696 InvalidEntry.Value);
697 }
698 }
699 }
700
701 if(AllocateL1)
702 {
703 uint64_t InvalidEntry = (!isTRVA) ? GMM_INVALID_AUX_ENTRY : (NullCCSTile | __BIT(0));
704 uint32_t i = 0;
705
706 if(DoNotWait)
707 {
708 GmmLib::LastLevelTable *pL1Tbl = NULL;
709 pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx, NULL);
710 L2TableCPUAdr = pTTL2[L3eIdx].GetCPUAddress();
711 L1TableCPUAdr = pL1Tbl->GetCPUAddress();
712 //Sync update on CPU
713 ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx].Value = 0;
714 GMM_TO_AUX_L2e_L1GFXADDR_2(L1TableAdr, ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx], (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext()))) // populate L2e.L1GfxAddr
715 ((GMM_AUXTTL2e *)L2TableCPUAdr)[L2eIdx]
716 .Valid = 1;
717 for(i = 0; i < (uint32_t)GMM_AUX_L1_SIZE(GetGmmLibContext()); i++)
718 {
719 //initialize L1e ie mark all entries with Null tile value
720 ((GMM_AUXTTL1e *)L1TableCPUAdr)[i].Value = InvalidEntry;
721 }
722 }
723 else
724 {
725 GMM_AUXTTL2e L2e = {0};
726 L2e.Valid = 1;
727 GMM_TO_AUX_L2e_L1GFXADDR_2(L1TableAdr, L2e, (!WA64K(GetGmmLibContext()) && !WA16K(GetGmmLibContext())))
728 pTTL2[L3eIdx]
729 .UpdatePoolFence(UmdContext, false);
730 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
731 L2TableAdr + L2eIdx * GMM_AUX_L2e_SIZE,
732 L2e.Value);
733
734 //initialize all L1e with invalid entries
735 for(i = 0; i < (uint32_t)GMM_AUX_L1_SIZE(GetGmmLibContext()); i++)
736 {
737 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
738 L1TableAdr + i * sizeof(uint64_t),
739 InvalidEntry);
740 }
741 }
742 }
743 }
744
745 //GMM_DPF(GFXDBG_NORMAL, "Mapping surface: GPUVA=0x%016llx Size=0x%08x Aux_GPUVA=0x%016llx", StartAdr, BaseSize, CCS$Adr);
746 for(TileAdr = StartAdr; TileAdr < EndAdr; TileAdr += (WA16K(GetGmmLibContext()) ? GMM_KBYTE(16) : WA64K(GetGmmLibContext()) ? GMM_KBYTE(64) : GMM_MBYTE(1)),
747 CCS$Adr += (pClientContext->GetLibContext()->GetSkuTable().FtrLinearCCS ?
748 (WA16K(pClientContext->GetLibContext()) ? GMM_BYTES(64) : WA64K(pClientContext->GetLibContext()) ? GMM_BYTES(256) : GMM_KBYTE(4)) :
749 0))
750 {
751 GMM_GFX_SIZE_T L1eIdx = GMM_L1_ENTRY_IDX(AUXTT, TileAdr, GetGmmLibContext());
752 GMM_AUXTTL1e L1e = {0};
753 L1e.Value = PartialData;
754 L1e.Valid = 1;
755
756 if(L1eIdx == 15 || L1eIdx == 14)
757 {
758 GMM_DPF(GFXDBG_NORMAL, "\n****** switching over L1*******\n");
759 }
760
761 CCS$Adr = (pClientContext->GetLibContext()->GetSkuTable().FtrLinearCCS ? CCS$Adr :
762 __GetCCSCacheline(BaseResInfo, BaseAdr, AuxResInfo, AuxVA, TileAdr - BaseAdr));
763
764 if(WA16K(GetGmmLibContext()))
765 {
766 L1e.Reserved2 = CCS$Adr >> 6; /*********** 2 lsbs of 64B-aligned CCS adr *****/
767 L1e.Reserved4 = CCS$Adr >> 8; /*********** 256B-aligned CCS adr *****/
768 L1e.GfxAddress = CCS$Adr >> 12; /*********** 4KB-aligned CCS adr *****/
769 }
770 else if(WA64K(GetGmmLibContext()))
771 {
772 __GMM_ASSERT((CCS$Adr & 0xFF) == 0x0);
773 __GMM_ASSERT(GFX_IS_ALIGNED(CCS$Adr, GMM_BYTES(256)));
774 __GMM_ASSERT(GFX_IS_ALIGNED(TileAdr, GMM_KBYTE(64)));
775 L1e.Reserved4 = CCS$Adr >> 8; /*********** 4 lsbs of 256B-aligned CCS adr *****/
776 L1e.GfxAddress = CCS$Adr >> 12; /*********** 4KB-aligned CCS adr *****/
777 }
778 else // 1MB aligned address
779 {
780 __GMM_ASSERT((CCS$Adr & 0xFF) == 0x0);
781 __GMM_ASSERT(GFX_IS_ALIGNED(CCS$Adr, GMM_KBYTE(4)));
782 __GMM_ASSERT(GFX_IS_ALIGNED(TileAdr, GMM_MBYTE(1)));
783 L1e.GfxAddress = CCS$Adr >> 12; /*********** 4KB-aligned CCS adr *****/
784 }
785
786
787 GMM_DPF(GFXDBG_NORMAL, "--------------------------------MAP AuxTT Map Address: TileAddr[0x%llX], Size[0x%x], CCSAddr[0x%llX], L3eIdx[%d], L2eIdx[%d], L1eIdx[%d], \n L1CCSAddres[0x%llX] \n", TileAdr, BaseSize, CCS$Adr, L3eIdx, L2eIdx, L1eIdx, L1e.GfxAddress);
788
789 GmmLib::LastLevelTable *pL1Tbl = NULL;
790
791 pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx, NULL);
792 L1TableCPUAdr = pL1Tbl->GetCPUAddress();
793 if(DoNotWait)
794 {
795 //Sync update on CPU
796 ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].Value = L1e.Value;
797
798 }
799 else
800 {
801 pL1Tbl->UpdatePoolFence(UmdContext, false);
802 PageTableMgr->TTCb.pfWriteL2L3Entry(UmdContext->pCommandQueueHandle,
803 L1TableAdr + L1eIdx * GMM_AUX_L1e_SIZE,
804 L1e.Value);
805 }
806 GMM_DPF(GFXDBG_NORMAL, "Map | L3 Table Entry: L3AddressBase[0x%llX] :: L3.L2GfxAddr[0x%llX] :: L3Valid[0x%llX] \n", (GMM_AUXTTL3e *)(TTL3.CPUAddress), ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].L2GfxAddr, ((GMM_AUXTTL3e *)(TTL3.CPUAddress))[L3eIdx].Valid);
807 GMM_DPF(GFXDBG_NORMAL, "Map | L2 Table Entry: L2addressBase[0x%llX] :: L2.L1GfxAddr[0x%llX] :: L2Valid[0x%llX] \n", ((GMM_AUXTTL2e *)pTTL2[L3eIdx].GetCPUAddress()), ((GMM_AUXTTL2e *)pTTL2[L3eIdx].GetCPUAddress())[L2eIdx].L1GfxAddr, ((GMM_AUXTTL2e *)pTTL2[L3eIdx].GetCPUAddress())[L2eIdx].Valid);
808 GMM_DPF(GFXDBG_NORMAL, "Map | L1 Table Entry: L1addressBase[0x%llX] :: L1.CCSAddr[0x%llX] :: L1ValueReserved4[0x%llX] ::L1ValueReserved2[0x%llX] :: L1Valid[0x%llX] :: DerivedCCS[0x%llX] \n\n", ((GMM_AUXTTL1e *)L1TableCPUAdr), ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].GfxAddress, ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].Reserved4, ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].Reserved2, ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].Valid, (((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].GfxAddress << 12));
809 GMM_DPF(GFXDBG_NORMAL, "**Map | Table Entry: L2addressBase[0x%llX] :: L2Valid[%d] :: L2eidx[%d] L1addressBase[0x%llX] :: L1eidx[%d] L1Valid[0x%llX] :: DerivedCCS[0x%llX]", ((GMM_AUXTTL2e *)pTTL2[L3eIdx].GetCPUAddress()), ((GMM_AUXTTL2e *)pTTL2[L3eIdx].GetCPUAddress())[L2eIdx].Valid, L2eIdx, GMM_L1TABLE_ADDR_FROM_AUX_L2e_L1GFXADDR(((GMM_AUXTTL2e *)pTTL2[L3eIdx].GetCPUAddress())[L2eIdx], true), L1eIdx, ((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].Valid, (((GMM_AUXTTL1e *)L1TableCPUAdr)[L1eIdx].GfxAddress << 12));
810
811 // Since we are mapping a non-null entry, no need to check whether
812 // L1 table is unused.
813 pL1Tbl->TrackTableUsage(AUXTT, true, TileAdr, false, GetGmmLibContext());
814 }
815 }
816 if(!DoNotWait)
817 {
818 PageTableMgr->TTCb.pfEpilogTranslationTable(
819 UmdContext->pCommandQueueHandle,
820 1);
821 }
822 }
823
824 LeaveCriticalSection(&TTLock);
825
826 return Status;
827 }
828
CreateAuxL1Data(GMM_RESOURCE_INFO * BaseResInfo)829 GMM_AUXTTL1e GmmLib::AuxTable::CreateAuxL1Data(GMM_RESOURCE_INFO *BaseResInfo)
830 {
831 GMM_RESOURCE_FORMAT Format;
832 Format = BaseResInfo->GetResourceFormat();
833 Format = ((Format > GMM_FORMAT_INVALID) && (Format < GMM_RESOURCE_FORMATS)) ? Format : GMM_FORMAT_INVALID;
834
835 GMM_FORMAT_ENTRY FormatInfo = pClientContext->GetLibContext()->GetPlatformInfo().FormatTable[Format];
836 GMM_AUXTTL1e L1ePartial = {0};
837 #define GMM_REGISTRY_UMD_PATH "SOFTWARE\\Intel\\IGFX\\GMM\\"
838 #define GMM_E2EC_OVERRIDEDEPTH16BPPTO12 "ForceYUV16To12BPP"
839
840 L1ePartial.Mode = BaseResInfo->GetResFlags().Info.RenderCompressed ? 0x1 : 0x0; //MC on VCS supports all compression modes,
841 //MC on Render pipe only 128B compr (until B-step)
842 //Recognize which .MC surfaces needs Render pipe access
843 if(pClientContext->GetLibContext()->GetWaTable().WaLimit128BMediaCompr)
844 {
845 L1ePartial.Mode = 0x1; //Limit media compression to 128B (same as RC) on gen12LP A0
846 }
847
848 //L1ePartial.Lossy = 0; // when to set it
849 L1ePartial.TileMode = BaseResInfo->GetResFlags().Info.TiledYs ? 0 : 1;
850
851 L1ePartial.Format = FormatInfo.CompressionFormat.AuxL1eFormat;
852 L1ePartial.LumaChroma = GmmIsPlanar(Format);
853
854 if(pClientContext->GetLibContext()->GetWaTable().WaUntypedBufferCompression && BaseResInfo->GetResourceType() == RESOURCE_BUFFER)
855 {
856 //Gen12LP WA to support untyped raw buffer compression on HDC ie MLC(machine-learning compression)
857 L1ePartial.TileMode = 0;
858 L1ePartial.Depth = 0x6;
859 L1ePartial.Format = GMM_E2ECOMP_FORMAT_RGBAFLOAT16;
860 }
861
862 __GMM_ASSERT(L1ePartial.Format > GMM_E2ECOMP_MIN_FORMAT && //Are we going to reuse 0x00 for uncompressed indication? CCS contains that info, but only known by HW
863 L1ePartial.Format <= GMM_E2ECOMP_MAX_FORMAT); //Could SW use it as surface-wide uncompressed state indicator? If so, remove teh assert (Need to make sure, all format encodings are correct)
864
865 if(BaseResInfo->GetResFlags().Info.RenderCompressed)
866 {
867 if(BaseResInfo->GetResourceType() != RESOURCE_BUFFER)
868 {
869 switch(FormatInfo.Element.BitsPer)
870 {
871 case 8:
872 L1ePartial.Depth = 0x4;
873 break;
874 case 16:
875 L1ePartial.Depth = 0x0;
876 break;
877 case 32:
878 L1ePartial.Depth = 0x5;
879 break;
880 case 64:
881 L1ePartial.Depth = 0x6;
882 break;
883 case 128:
884 L1ePartial.Depth = 0x7;
885 break;
886 default:
887 L1ePartial.Depth = 0x3;
888 }
889 }
890 }
891 else
892 {
893 switch(Format)
894 {
895 case GMM_FORMAT_P012:
896 case GMM_FORMAT_Y412:
897 case GMM_FORMAT_Y212: //which format encoding for Y212, Y412, P012?
898 L1ePartial.Depth = 0x2;
899 break;
900 case GMM_FORMAT_P010:
901 //case GMM_FORMAT_Y410:
902 case GMM_FORMAT_Y210: //which format encoding for Y210?
903 L1ePartial.Depth = 0x1;
904 break;
905 case GMM_FORMAT_P016: //per HAS, separate encoding than P010, but a comment says to use P010 in AuxTable?
906 case GMM_FORMAT_Y416:
907 case GMM_FORMAT_Y216:
908 L1ePartial.Depth = 0x0;
909 break;
910 default:
911 L1ePartial.Depth = 0x3; //For MC, bpp got from format encoding
912 }
913
914 if(L1ePartial.Format == GMM_E2ECOMP_FORMAT_R10G10B10A2_UNORM)
915 {
916 L1ePartial.Format = GMM_E2ECOMP_FORMAT_RGB10b;
917 }
918 }
919
920 return L1ePartial;
921 }
922
__GetCCSCacheline(GMM_RESOURCE_INFO * BaseResInfo,GMM_GFX_ADDRESS BaseAdr,GMM_RESOURCE_INFO * AuxResInfo,GMM_GFX_ADDRESS AuxVA,GMM_GFX_SIZE_T AdrOffset)923 GMM_GFX_ADDRESS GMM_INLINE GmmLib::AuxTable::__GetCCSCacheline(GMM_RESOURCE_INFO *BaseResInfo, GMM_GFX_ADDRESS BaseAdr,
924 GMM_RESOURCE_INFO *AuxResInfo, GMM_GFX_ADDRESS AuxVA, GMM_GFX_SIZE_T AdrOffset)
925 {
926 GMM_GFX_ADDRESS CCSChunkAdr = 0xFFFFFFF0;
927 uint32_t x = 0, y = 0;
928 uint32_t i = 0, j = 0;
929 uint32_t CCSXTile = 0, CCSYTile = 0;
930 GMM_UNREFERENCED_PARAMETER(BaseAdr);
931
932 bool BaseIsYF = BaseResInfo->GetResFlags().Info.TiledYf ? true : false;
933 uint32_t BasePitchInTiles = BaseResInfo->GetRenderPitchTiles();
934
935 //Find YF/YS TileId <x,y> for given main surface 16K-chunk
936 //and CCS$Id <i,j> corresponding to main's <x,y>
937 AdrOffset >>= 14; //AdrOffset must be 16K-aligned chunk, since mapping unit is 4 YF pages
938 if(BaseIsYF)
939 {
940 uint32_t PitchIn4YF = BasePitchInTiles / 4; //Base Pitch is physically padded to 4x1 YF width
941
942 if (PitchIn4YF != 0)
943 {
944 i = static_cast<uint32_t>(AdrOffset % PitchIn4YF);
945 j = static_cast<uint32_t>(AdrOffset / PitchIn4YF);
946 }
947 else
948 {
949 __GMM_ASSERT(PitchIn4YF != 0);
950 return 0;
951 }
952 }
953 else if(BasePitchInTiles != 0) //TileYs
954 {
955 x = static_cast<uint32_t>(AdrOffset >> 2); //YS-tile count
956 y = x / BasePitchInTiles; //YS- tile id <x,y>
957 x = x % BasePitchInTiles;
958 i = 2 * x;
959 j = 2 * y;
960 switch(AdrOffset % 4) //YS : XYXY [XYXY YF] ie 2x2 16K-units in Y-major
961 {
962 case 0:
963 break;
964 case 1:
965 j++;
966 break;
967 case 2:
968 i++;
969 break;
970 case 3:
971 i++;
972 j++;
973 break;
974 }
975 }
976
977 //Compute CCS$ address for <i,j>
978 CCSXTile = (i >= 8) ? i / 8 : 0; //8x8 CLs make one CCS Tile; get TileOffset
979 CCSYTile = (j >= 8) ? j / 8 : 0;
980 i %= 8;
981 j %= 8;
982
983 uint32_t AuxPitchInTiles = AuxResInfo ? AuxResInfo->GetRenderPitchTiles() : BaseResInfo->GetRenderAuxPitchTiles();
984 CCSChunkAdr = AuxVA + ((CCSXTile + CCSYTile * AuxPitchInTiles) * GMM_KBYTE(4)) + (8 * GMM_BYTES(64) * i) + (GMM_BYTES(64) * j);
985
986 return CCSChunkAdr;
987 }
988
989 #endif /*!__GMM_KMD__*/
990