1 /*==============================================================================
2 Copyright(c) 2019 Intel Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a
5 copy of this software and associated documentation files(the "Software"),
6 to deal in the Software without restriction, including without limitation
7 the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and / or sell copies of the Software, and to permit persons to whom the
9 Software is furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included
12 in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 OTHER DEALINGS IN THE SOFTWARE.
21
22 Description: Contains functions of internal classes
23 (ie PageTablePool, PageTable, Table), that support
24 user mode page table management
25
26 ============================================================================*/
27
28 #include "Internal/Common/GmmLibInc.h"
29 #include "../TranslationTable/GmmUmdTranslationTable.h"
30 #include "Internal/Common/Texture/GmmTextureCalc.h"
31
32 #if !defined(__GMM_KMD)
33
34 #if defined(__linux__)
35 #include "Internal/Linux/GmmResourceInfoLinInt.h"
36
37 #define _aligned_free(ptr) free(ptr)
38
39 #endif
40
41 //=============================================================================
42 //
43 // Function: AllocateL3Table
44 //
45 // Desc: Allocates (always resident SVM) memory for AUXTT\L3 Table, and updates AUXTT object
46 //
47 // Parameters:
48 // pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
49 //
50 // Returns:
51 // GMM_SUCCESS on success,
52 // GMM_INVALIDPARAM on invalid parameter(s)
53 // GMM_OUT_OF_MEMORY on memory allocation failure, failure to make resident
54 //-----------------------------------------------------------------------------
AllocateL3Table(uint32_t L3TableSize,uint32_t L3AddrAlignment)55 GMM_STATUS GmmLib::PageTable::AllocateL3Table(uint32_t L3TableSize, uint32_t L3AddrAlignment)
56 {
57 GMM_STATUS Status = GMM_SUCCESS;
58 GMM_DEVICE_ALLOC Alloc = {0};
59
60 __GMM_ASSERTPTR(PageTableMgr, GMM_INVALIDPARAM);
61
62 EnterCriticalSection(&TTLock);
63
64 Alloc.Size = L3TableSize;
65 Alloc.Alignment = L3AddrAlignment;
66 Alloc.hCsr = PageTableMgr->hCsr;
67
68 Status = __GmmDeviceAlloc(pClientContext, &PageTableMgr->DeviceCbInt, &Alloc);
69 if(Status != GMM_SUCCESS)
70 {
71 LeaveCriticalSection(&TTLock);
72 return Status;
73 }
74
75 TTL3.GfxAddress = GMM_GFX_ADDRESS_CANONIZE(Alloc.GfxVA);
76 TTL3.CPUAddress = Alloc.CPUVA;
77 TTL3.NeedRegisterUpdate = true;
78 TTL3.L3Handle = (HANDLE)(uintptr_t)Alloc.Handle;
79 TTL3.pGmmResInfo = (GMM_RESOURCE_INFO *)Alloc.Priv;
80
81 // Invalidate L3e's
82 for(int i = 0; i < (GMM_L3_SIZE(TTType)); i++)
83 {
84 //initialize L3e ie mark all entries with Null tile/invalid value
85 ((GMM_AUXTTL3e *)TTL3.CPUAddress)[i].Value = 0;
86
87 }
88
89 LeaveCriticalSection(&TTLock);
90 return Status;
91 }
92
93 //=============================================================================
94 //
95 // Function: __IsUnusedTRTTPoolOverLimit
96 //
97 // Desc: Checks if unused TRTTPools have reached residency limit and must be freed.
98 //
99 // Parameters:
100 // pTRTT_Obj: per-device TT object. Contains TT node info
101 // OverLimitSize: Size in bytes that can be freed
102 //
103 // Returns:
104 // True, if unused TTPool reached max. residency limit
105 // False, otherwise
106 //-----------------------------------------------------------------------------
__IsUnusedTRTTPoolOverLimit(GMM_GFX_SIZE_T * OverLimitSize)107 bool GmmLib::GmmPageTablePool::__IsUnusedTRTTPoolOverLimit(GMM_GFX_SIZE_T *OverLimitSize)
108 {
109 GMM_GFX_SIZE_T UnusedTrTTPoolSize = 0;
110 GmmLib::GMM_PAGETABLEPool *Pool = NULL;
111
112 Pool = this;
113
114 while(Pool)
115 {
116 if(Pool->NumFreeNodes == PAGETABLE_POOL_MAX_NODES)
117 {
118 UnusedTrTTPoolSize += PAGETABLE_POOL_SIZE;
119 }
120 Pool = Pool->NextPool;
121 }
122
123 *OverLimitSize = (UnusedTrTTPoolSize > PAGETABLE_POOL_MAX_UNUSED_SIZE) ? (UnusedTrTTPoolSize - PAGETABLE_POOL_MAX_UNUSED_SIZE) : 0;
124 return (UnusedTrTTPoolSize > PAGETABLE_POOL_MAX_UNUSED_SIZE) ? true : false;
125 }
126
127 //=============================================================================
128 //
129 // Function: AllocateL1L2Table
130 //
131 // Desc: Assigns pages from AUXTTPool for L1/L2Table for translation of given TRVA
132 //
133 // Parameters:
134 // pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
135 // TileAddr: Tiled Resource Virtual address
136 //
137 // Returns:
138 // L2Table/L1Table Address
139 //-----------------------------------------------------------------------------
AllocateL1L2Table(GMM_GFX_ADDRESS TileAddr,GMM_GFX_ADDRESS * L1TableAdr,GMM_GFX_ADDRESS * L2TableAdr)140 void GmmLib::PageTable::AllocateL1L2Table(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS *L1TableAdr, GMM_GFX_ADDRESS *L2TableAdr)
141 {
142 GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
143 uint32_t L3eIdx = static_cast<uint32_t>(GMM_L3_ENTRY_IDX(TTType, TileAddr));
144 uint32_t L2eIdx = static_cast<uint32_t>(GMM_L2_ENTRY_IDX(TTType, TileAddr));
145 GmmLib::LastLevelTable *pL1Tbl = NULL;
146
147 __GMM_ASSERTPTR(pClientContext, VOIDRETURN); // void return
148
149 *L2TableAdr = GMM_NO_TABLE;
150 *L1TableAdr = GMM_NO_TABLE;
151
152 if(TTL3.L3Handle)
153 {
154 L3TableAdr = TTL3.GfxAddress;
155 }
156 else
157 {
158 //Should never hit -- L3Table is allocated during device creation
159 __GMM_ASSERT(false);
160 }
161
162 if(pTTL2[L3eIdx].GetPool())
163 {
164 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
165 PoolElem = pTTL2[L3eIdx].GetPool();
166 *L2TableAdr = (PoolElem != NULL) ? PoolElem->GetGfxAddress() + (PAGE_SIZE * pTTL2[L3eIdx].GetNodeIdx()) : GMM_NO_TABLE;
167 }
168 else
169 {
170 uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
171 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
172 POOL_TYPE PoolType = POOL_TYPE_AUXTTL2;
173 PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType);
174 if(PoolElem)
175 {
176 pTTL2[L3eIdx] = MidLevelTable(PoolElem, PoolNodeIdx, PoolElem->GetNodeBBInfoAtIndex(PoolNodeIdx));
177 *L2TableAdr = PoolElem->GetGfxAddress() + PAGE_SIZE * PoolNodeIdx; //PoolNodeIdx must be multiple of 8 (Aux L2) and multiple of 2 (Aux L1)
178 ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, NodesPerTable)
179 }
180 }
181
182 pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
183 if(pL1Tbl)
184 {
185 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
186 PoolElem = pL1Tbl->GetPool();
187 *L1TableAdr = (PoolElem != NULL) ? PoolElem->GetGfxAddress() + (PAGE_SIZE * pL1Tbl->GetNodeIdx()) : GMM_NO_TABLE;
188 }
189 else
190 {
191 //Allocate L1 Table
192 uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
193 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
194 POOL_TYPE PoolType = POOL_TYPE_AUXTTL1;
195
196 PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType); //Recognize if Aux-L1 being allocated
197 if(PoolElem)
198 {
199 pL1Tbl = new GmmLib::LastLevelTable(PoolElem, PoolNodeIdx, GMM_L1_SIZE_DWORD(TTType, GetGmmLibContext()), L2eIdx); // use TR vs Aux L1_Size_DWORD
200
201 if(pL1Tbl)
202 {
203 *L1TableAdr = PoolElem->GetGfxAddress() + PAGE_SIZE * PoolNodeIdx; //PoolNodeIdx should reflect 1 node per Tr-table and 2 nodes per AUX L1 TABLE
204 if(PoolNodeIdx != PAGETABLE_POOL_MAX_NODES)
205 {
206 uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES_2(GetGmmLibContext()) : 1;
207 ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, PerTableNodes)
208 }
209 pTTL2[L3eIdx].InsertInList(pL1Tbl);
210 }
211 }
212 }
213 }
214
215 //=============================================================================
216 //
217 // Function: AllocateDummyTables
218 //
219 // Desc: Assigns pages from AUXTTPool for Dummy L1/L2Table
220 //
221 // Parameters:
222 // L2Table - Ptr to initiatize dummy L2Table
223 // L1Table - Ptr to initiatize dummy L1Table
224 //
225 // Returns:
226 // L2Table/L1Tables
227 //-----------------------------------------------------------------------------
AllocateDummyTables(GmmLib::Table ** L2Table,GmmLib::Table ** L1Table)228 void GmmLib::PageTable::AllocateDummyTables(GmmLib::Table **L2Table, GmmLib::Table **L1Table)
229 {
230 GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
231 GmmLib::LastLevelTable *pL1Tbl = NULL;
232
233 __GMM_ASSERTPTR(pClientContext, VOIDRETURN);
234
235 if(TTL3.L3Handle)
236 {
237 L3TableAdr = TTL3.GfxAddress;
238 }
239 else
240 {
241 //Should never hit -- L3Table is allocated during device creation
242 __GMM_ASSERT(false);
243 }
244
245 //Allocate dummy L2Table
246 {
247 uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
248 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
249 POOL_TYPE PoolType = POOL_TYPE_AUXTTL2;
250 PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType);
251 if(PoolElem)
252 {
253 *L2Table = new GmmLib::MidLevelTable(PoolElem, PoolNodeIdx, PoolElem->GetNodeBBInfoAtIndex(PoolNodeIdx));
254 ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, NodesPerTable)
255 }
256 }
257
258 //Allocate dummy L1Table
259 {
260 uint32_t PoolNodeIdx = PAGETABLE_POOL_MAX_NODES;
261 GmmLib::GMM_PAGETABLEPool *PoolElem = NULL;
262 POOL_TYPE PoolType = POOL_TYPE_AUXTTL1;
263
264 PoolElem = PageTableMgr->__GetFreePoolNode(&PoolNodeIdx, PoolType); //Recognize if Aux-L1 being allocated
265 if(PoolElem)
266 {
267 *L1Table = new GmmLib::LastLevelTable(PoolElem, PoolNodeIdx, GMM_L1_SIZE_DWORD(TTType, GetGmmLibContext()), 0); // use TR vs Aux L1_Size_DWORD
268
269 if(*L1Table)
270 {
271 if(PoolNodeIdx != PAGETABLE_POOL_MAX_NODES)
272 {
273 uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES_2(GetGmmLibContext()) : 1;
274 ASSIGN_POOLNODE(PoolElem, PoolNodeIdx, PerTableNodes)
275 }
276 }
277 }
278 }
279 }
280
281 //=============================================================================
282 //
283 // Function: GetL1L2TableAddr
284 //
285 // Desc: For given tile address, returns L1/L2 Table address if the table exists
286 //
287 // Parameters:
288 // pAUXTT_Obj: per-device AUX-TT object. Contains AXUTT node info
289 // TileAddr: Tiled Resource Virtual address
290 //
291 // Returns:
292 // L2Table/L1Table Address
293 //-----------------------------------------------------------------------------
GetL1L2TableAddr(GMM_GFX_ADDRESS TileAddr,GMM_GFX_ADDRESS * L1TableAdr,GMM_GFX_ADDRESS * L2TableAdr)294 void GmmLib::PageTable::GetL1L2TableAddr(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS *L1TableAdr, GMM_GFX_ADDRESS *L2TableAdr)
295 {
296 GMM_GFX_SIZE_T L3eIdx, L2eIdx, L1eIdx;
297 GMM_GFX_ADDRESS L3TableAdr = GMM_NO_TABLE;
298 *L2TableAdr = GMM_NO_TABLE;
299 *L1TableAdr = GMM_NO_TABLE;
300
301 __GMM_ASSERTPTR(pClientContext, VOIDRETURN);
302
303 L3eIdx = GMM_L3_ENTRY_IDX(TTType, TileAddr);
304 L2eIdx = GMM_L2_ENTRY_IDX(TTType, TileAddr);
305 L1eIdx = GMM_L1_ENTRY_IDX(TTType, TileAddr, GetGmmLibContext());
306
307 __GMM_ASSERT(TTL3.L3Handle);
308 L3TableAdr = TTL3.GfxAddress;
309 if(pTTL2[L3eIdx].GetPool())
310 {
311 GmmLib::GMM_PAGETABLEPool *Pool = NULL;
312 GmmLib::LastLevelTable * pL1Tbl = NULL;
313 Pool = pTTL2[L3eIdx].GetPool();
314
315 if(Pool)
316 {
317 __GMM_ASSERT(Pool->GetNumFreeNode() != PAGETABLE_POOL_MAX_NODES);
318 __GMM_ASSERT(pTTL2[L3eIdx].GetNodeIdx() < PAGETABLE_POOL_MAX_NODES);
319 __GMM_ASSERT(Pool->GetNodeUsageAtIndex(pTTL2[L3eIdx].GetNodeIdx() / (32 * NodesPerTable)) != 0);
320
321 *L2TableAdr = Pool->GetGfxAddress() + PAGE_SIZE * (pTTL2[L3eIdx].GetNodeIdx());
322 }
323
324 pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
325 if(pL1Tbl && pL1Tbl->GetPool())
326 {
327 Pool = NULL;
328 Pool = pL1Tbl->GetPool();
329 if(Pool)
330 {
331 uint32_t PerTableNodes = (TTType == AUXTT) ? AUX_L1TABLE_SIZE_IN_POOLNODES_2(GetGmmLibContext()) : 1;
332 __GMM_ASSERT(Pool->GetNumFreeNode() != PAGETABLE_POOL_MAX_NODES);
333 __GMM_ASSERT(pL1Tbl->GetNodeIdx() < PAGETABLE_POOL_MAX_NODES);
334 __GMM_ASSERT(Pool->GetNodeUsageAtIndex(pL1Tbl->GetNodeIdx() / (32 * PerTableNodes)) != 0);
335
336 *L1TableAdr = Pool->GetGfxAddress() + PAGE_SIZE * (pL1Tbl->GetNodeIdx());
337 }
338 }
339 }
340 }
341
342 //=============================================================================
343 //
344 // Function: GetMappingType
345 //
346 // Desc: For given gfx address and size, returns MappingType (null/non-null or
347 // Valid/Invalid) of GfxVA and first gfx address have reverse mapping
348 //
349 /// @param[in] GfxVA: Gfx Address whose mapping type is being queried
350 /// @param[in] Size: Size of interested Gfx address range
351 /// @param[out] LastAddr : 1st Gfx Address having reverse mapping type
352 //
353 /// @return 1/0 : for non-null/valid vs null/invalid mapping
354 //-----------------------------------------------------------------------------
GetMappingType(GMM_GFX_ADDRESS GfxVA,GMM_GFX_SIZE_T Size,GMM_GFX_ADDRESS & LastAddr)355 uint8_t GmmLib::PageTable::GetMappingType(GMM_GFX_ADDRESS GfxVA, GMM_GFX_SIZE_T Size, GMM_GFX_ADDRESS &LastAddr)
356 {
357 GMM_GFX_SIZE_T L3eIdx, L2eIdx, L1eIdx, L1EntrySize;
358 uint32_t L1Size, L2Size;
359 uint8_t MapType = 0; //true for non-null, false for null mapped
360 bool bFoundLastVA = false, bTerminate = false;
361 GMM_GFX_ADDRESS TileAddr = GfxVA;
362
363 L3eIdx = GMM_L3_ENTRY_IDX(TTType, GfxVA);
364 L2eIdx = GMM_L2_ENTRY_IDX(TTType, GfxVA);
365 L1eIdx = GMM_L1_ENTRY_IDX(TTType, GfxVA, GetGmmLibContext());
366 L1EntrySize = WA16K(GetGmmLibContext()) ? GMM_KBYTE(16) : WA64K(GetGmmLibContext()) ? GMM_KBYTE(64) : GMM_MBYTE(1);
367
368 EnterCriticalSection(&TTLock);
369 __GMM_ASSERT(TTL3.L3Handle);
370
371 #define GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx) \
372 { \
373 L1eIdx = 0; \
374 L2eIdx++; \
375 if(L2eIdx == (GMM_L2_SIZE(TTType))) \
376 { \
377 L2eIdx = 0; \
378 L3eIdx++; \
379 if(L3eIdx == (GMM_L3_SIZE(TTType))) \
380 { \
381 bTerminate = true; \
382 } \
383 } \
384 }
385
386 #define GET_NEXT_L2TABLE(L1eIdx, L2eIdx, L3eIdx) \
387 { \
388 L1eIdx = L2eIdx = 0; \
389 L3eIdx++; \
390 if(L3eIdx == (GMM_L3_SIZE(TTType))) \
391 { \
392 bTerminate = true; \
393 } \
394 }
395
396 while(!(bFoundLastVA || bTerminate) && (TileAddr < GfxVA + Size))
397 {
398 if(pTTL2[L3eIdx].GetPool())
399 {
400 GmmLib::LastLevelTable *pL1Tbl = NULL;
401 pL1Tbl = pTTL2[L3eIdx].GetL1Table(L2eIdx);
402 if(pL1Tbl && pL1Tbl->GetPool())
403 {
404 uint32_t LastBit = 0;
405 uint32_t i = static_cast<uint32_t>(L1eIdx) / 32;
406
407 while(!bFoundLastVA && i < (uint32_t)(GMM_L1_SIZE_DWORD(TTType, GetGmmLibContext())))
408 {
409 uint32_t UsageDW = pL1Tbl->GetUsedEntries()[i++];
410 uint32_t BitNum = 31;
411 if(GfxVA == TileAddr)
412 {
413 BitNum = L1eIdx % 32;
414 MapType = ((UsageDW & __BIT(BitNum)) ? 0x1 : 0x0); //true for non-null, false for null mapped
415 UsageDW = (!MapType) ? UsageDW : ~UsageDW;
416 UsageDW = ((uint64_t)UsageDW >> (BitNum + 1)) << (BitNum + 1); // clear lsb <= BitNum
417 }
418 else
419 {
420 UsageDW = (!MapType) ? UsageDW : ~UsageDW;
421 }
422
423 if(_BitScanForward((uint32_t *)&LastBit, UsageDW)) // Gets lsb > BitNum, having reverse mapType
424 {
425 bFoundLastVA = true;
426 uint32_t NumTiles = (GfxVA == TileAddr) ? (LastBit - BitNum) : LastBit;
427 LastAddr = TileAddr + NumTiles * L1EntrySize;
428 }
429 else
430 {
431 uint32_t NumTiles = (GfxVA == TileAddr) ? (32 - BitNum) : 32;
432 TileAddr += NumTiles * L1EntrySize;
433 }
434 }
435 if(!bFoundLastVA)
436 {
437 GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx);
438 }
439 }
440 else //L2Entry is NULL
441 {
442 if(MapType) //First hit null-map
443 {
444 LastAddr = TileAddr;
445 bFoundLastVA = true;
446 }
447 else
448 {
449 GMM_GFX_SIZE_T NumTiles = GMM_L1_USABLESIZE(TTType, GetGmmLibContext());
450 if(GfxVA == TileAddr)
451 {
452 MapType = false;
453 NumTiles -= L1eIdx;
454 }
455 TileAddr += NumTiles * L1EntrySize;
456 GET_NEXT_L1TABLE(L1eIdx, L2eIdx, L3eIdx)
457 }
458 }
459 }
460 else //L3entry is NULL
461 {
462 if(MapType) //First hit null-map
463 {
464 LastAddr = TileAddr;
465 bFoundLastVA = true;
466 }
467 else
468 {
469 GMM_GFX_SIZE_T NumTiles = 0;
470 if(GfxVA == TileAddr)
471 {
472 MapType = false;
473 L1Size = GMM_L1_USABLESIZE(TTType, GetGmmLibContext()) - L1eIdx;
474 L2Size = GMM_L2_SIZE(TTType) - L2eIdx;
475 NumTiles = ((uint64_t)L1Size * L2Size);
476 }
477 else
478 {
479 L1Size = GMM_L1_USABLESIZE(TTType, GetGmmLibContext());
480 L2Size = GMM_L2_SIZE(TTType);
481 NumTiles = ((uint64_t)L1Size * L2Size);
482 }
483 TileAddr += NumTiles * L1EntrySize;
484 GET_NEXT_L2TABLE(L1eIdx, L2eIdx, L3eIdx)
485 }
486 }
487 }
488
489 if(!bFoundLastVA)
490 {
491 LastAddr = TileAddr;
492 }
493
494 LeaveCriticalSection(&TTLock);
495 return MapType;
496 }
497
498 //=============================================================================
499 //
500 // Function: TrackTableUsage
501 //
502 // Desc: For given tile address, updates Table Usage.If Table has all Nullmappings
503 // then its pool node can be unassigned
504 //
505 // Parameters:
506 // Type: Translation Table type (Aux)
507 // IsL1: Is called for L1table or L2 Table
508 // TileAddr: Tiled Resource Virtual address
509 // NullMapped: true if given tiled adr was null mapped, otherwise false
510 //
511 // Returns:
512 // true, if Table for given tile adr is all null mapped
513 // false,if Table does not exist or has non-null mapping
514 //-----------------------------------------------------------------------------
TrackTableUsage(TT_TYPE Type,bool IsL1,GMM_GFX_ADDRESS TileAdr,bool NullMapped,GMM_LIB_CONTEXT * pGmmLibContext)515 bool GmmLib::Table::TrackTableUsage(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr, bool NullMapped, GMM_LIB_CONTEXT *pGmmLibContext )
516 {
517 uint32_t EntryIdx;
518 uint32_t ElemNum = 0, BitNum = 0;
519
520 EntryIdx = IsL1 ? static_cast<uint32_t>(GMM_L1_ENTRY_IDX(Type, TileAdr, pGmmLibContext)) : static_cast<uint32_t>(GMM_L2_ENTRY_IDX(Type, TileAdr));
521
522 ElemNum = EntryIdx / (sizeof(UsedEntries[0]) * 8);
523 BitNum = EntryIdx % (sizeof(UsedEntries[0]) * 8);
524
525 if(NullMapped)
526 {
527 UsedEntries[ElemNum] &= ~(1 << BitNum);
528 }
529 else
530 {
531 UsedEntries[ElemNum] |= (1 << BitNum);
532 }
533
534 if(NullMapped)
535 {
536 int TableDWSize = IsL1 ? static_cast<int>(GMM_L1_SIZE_DWORD(Type, pGmmLibContext)) : static_cast<int>(GMM_L2_SIZE_DWORD(Type));
537 for(int i = 0; i < TableDWSize; i++)
538 {
539 if(UsedEntries[i])
540 {
541 return false;
542 }
543 }
544 }
545 return NullMapped ? true : false;
546 }
547
548 //=============================================================================
549 //
550 // Function: __IsTableNullMapped
551 //
552 // Desc: For given tile address, checks if given Table has all Nullmappings
553 // then its pool node can be unassigned
554 //
555 // Parameters:
556 // Type: Translation Table type (TR or Aux)
557 // IsL1: Is called for L1table or L2 Table
558 // TileAddr: Tiled Resource Virtual address
559 //
560 // Returns:
561 // true, if Table for given tile adr is all null mapped
562 // false,if Table has non-null mapping
563 //-----------------------------------------------------------------------------
IsTableNullMapped(TT_TYPE Type,bool IsL1,GMM_GFX_ADDRESS TileAdr,GMM_LIB_CONTEXT * pGmmLibContext)564 bool GmmLib::Table::IsTableNullMapped(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr, GMM_LIB_CONTEXT *pGmmLibContext)
565 {
566 GMM_UNREFERENCED_PARAMETER(TileAdr);
567 int TableDWSize = IsL1 ? static_cast<int>(GMM_L1_SIZE_DWORD(Type, pGmmLibContext)) : static_cast<int>(GMM_L2_SIZE_DWORD(Type));
568 for(int i = 0; i < TableDWSize; i++)
569 {
570 if(UsedEntries[i])
571 {
572 return false;
573 }
574 }
575 return true;
576 }
577
578 //=============================================================================
579 //
580 // Function: __UpdatePoolFence
581 //
582 // Desc: Updates AUXTTPool's or Table's BBFenceObj/value with current BB fence
583 //
584 // Parameters:
585 // pAUXTT_Obj: per-device AUX-TT object. Contains AUXTT node info
586 // Table: L1/L2 table pointer
587 // L1Table: true for L1 Table, else false
588 // ClearNode: if true, Fence info is cleared for table
589 // false, Fence info is updated for table and pool
590 //-----------------------------------------------------------------------------
UpdatePoolFence(GMM_UMD_SYNCCONTEXT * UmdContext,bool ClearNode)591 void GmmLib::Table::UpdatePoolFence(GMM_UMD_SYNCCONTEXT *UmdContext, bool ClearNode)
592 {
593 if(!ClearNode)
594 {
595 //update both node and pool with current fence/handle
596 PoolElem->GetPoolBBInfo().BBQueueHandle =
597 BBInfo.BBQueueHandle = UmdContext->BBFenceObj;
598 PoolElem->GetPoolBBInfo().BBFence =
599 BBInfo.BBFence = UmdContext->BBLastFence + 1; //Save incremented fence value, since DX does it during submission
600 }
601 else
602 {
603 //Clear node fence/handle
604 BBInfo.BBQueueHandle = 0;
605 BBInfo.BBFence = 0;
606 }
607 }
608
609 /////////////////////////////////////////////////////////////////////////////////////
610 /// Releases all PageTable Pool(s) existing in Linked List
611 ///
612 /// @param[in] DeviceCallbacks pointer to device callbacks structure
613 /// @return GMM_STATUS
614 /////////////////////////////////////////////////////////////////////////////////////
__DestroyPageTablePool(void * DeviceCallbacks,HANDLE hCsr)615 GMM_STATUS GmmLib::GmmPageTablePool::__DestroyPageTablePool(void *DeviceCallbacks, HANDLE hCsr)
616 {
617 GMM_STATUS Status = GMM_SUCCESS;
618 GMM_DEVICE_CALLBACKS_INT *DeviceCb = static_cast<GMM_DEVICE_CALLBACKS_INT *>(DeviceCallbacks);
619
620 GMM_PAGETABLEPool *Node = this, *Next = NULL;
621 GMM_CLIENT ClientType;
622 GMM_DEVICE_DEALLOC Dealloc = {0};
623
624 //Evict/Free gpu Va is implictly done by OS when de-allocating
625 while(Node)
626 {
627 Next = Node->NextPool;
628
629 GET_GMM_CLIENT_TYPE(Node->pClientContext, ClientType);
630
631 Dealloc.Handle = Node->PoolHandle;
632 Dealloc.GfxVA = Node->PoolGfxAddress;
633 Dealloc.Priv = Node->pGmmResInfo;
634 Dealloc.hCsr = hCsr;
635
636 Status = __GmmDeviceDealloc(ClientType, DeviceCb, &Dealloc, Node->pClientContext);
637
638 Node->PoolHandle = NULL;
639 Node->PoolGfxAddress = 0;
640 delete Node;
641 Node = Next;
642 }
643
644 return Status;
645 }
646
647 /////////////////////////////////////////////////////////////////////////////////////
648 /// Releases memory allocated to PageTable's Root-table
649 ///
650 /// @return GMM_STATUS
651 /////////////////////////////////////////////////////////////////////////////////////
DestroyL3Table()652 GMM_STATUS GmmLib::PageTable::DestroyL3Table()
653 {
654 GMM_STATUS Status = GMM_SUCCESS;
655 uint8_t hr = GMM_SUCCESS;
656 GMM_CLIENT ClientType;
657 GMM_DEVICE_DEALLOC Dealloc = {0};
658
659 GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
660
661 EnterCriticalSection(&TTLock);
662
663 if(TTL3.L3Handle)
664 {
665 Dealloc.Handle = TTL3.L3Handle;
666 Dealloc.GfxVA = TTL3.GfxAddress;
667 Dealloc.Priv = TTL3.pGmmResInfo;
668 Dealloc.hCsr = PageTableMgr->hCsr;
669
670 Status = __GmmDeviceDealloc(ClientType, &PageTableMgr->DeviceCbInt, &Dealloc, pClientContext);
671
672 TTL3.L3Handle = NULL;
673 TTL3.GfxAddress = 0;
674 TTL3.CPUAddress = 0;
675 }
676
677 LeaveCriticalSection(&TTLock);
678 return Status;
679 }
680
681 #endif //!__GMM_KMD
682