1 /*==============================================================================
2 Copyright(c) 2019 Intel Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a
5 copy of this software and associated documentation files(the "Software"),
6 to deal in the Software without restriction, including without limitation
7 the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and / or sell copies of the Software, and to permit persons to whom the
9 Software is furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included
12 in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 OTHER DEALINGS IN THE SOFTWARE.
21
22 Description: This file contains the class definitions for GmmPageTablePool
23 PageTable, and low-level Tables for user-mode PageTable management,
24 that is common for both Linux and Windows.
25
26 ======================= end_copyright_notice ==================================*/
27 #pragma once
28 #include "External/Common/GmmPageTableMgr.h"
29
30 #ifdef __linux__
31 #include <pthread.h>
32 #include <string.h>
33
34 // Internal Linux version of MSDK APIs.
InitializeCriticalSection(pthread_mutex_t * mutex)35 static inline void InitializeCriticalSection(pthread_mutex_t *mutex)
36 {
37 pthread_mutexattr_t Attr;
38 pthread_mutexattr_init(&Attr);
39 pthread_mutexattr_settype(&Attr, PTHREAD_MUTEX_RECURSIVE);
40 pthread_mutex_init(mutex, &Attr);
41 }
42
DeleteCriticalSection(pthread_mutex_t * mutex)43 static inline void DeleteCriticalSection(pthread_mutex_t *mutex)
44 {
45 pthread_mutex_destroy(mutex);
46 }
47
EnterCriticalSection(pthread_mutex_t * mutex)48 static inline void EnterCriticalSection(pthread_mutex_t *mutex)
49 {
50 pthread_mutex_lock(mutex);
51 }
52
LeaveCriticalSection(pthread_mutex_t * mutex)53 static inline void LeaveCriticalSection(pthread_mutex_t *mutex)
54 {
55 pthread_mutex_unlock(mutex);
56 }
57
58 #ifndef _BitScanForwardDefined
_BitScanForward(uint32_t * index,uint32_t mask)59 static inline int _BitScanForward(uint32_t *index, uint32_t mask)
60 {
61 int i;
62
63 #ifdef __ANDROID__
64 i = ffs(mask);
65 #else
66 i = ffsl(mask);
67 #endif
68
69 if(i > 0)
70 {
71 *index = (uint32_t)(i - 1);
72 return i;
73 }
74 return 0;
75 }
76 #endif
77 #endif
78
79 #define GMM_L1_USABLESIZE(TTType, pGmmLibContext) (GMM_AUX_L1_USABLESIZE(pGmmLibContext))
80 #define GMM_L1_SIZE(TTType, pGmmLibContext) (GMM_AUX_L1_SIZE(pGmmLibContext))
81 #define GMM_L1_SIZE_DWORD(TTType, pGmmLibContext) (GMM_AUX_L1_SIZE_DWORD(pGmmLibContext))
82 #define GMM_L2_SIZE(TTType) (GMM_AUX_L2_SIZE)
83 #define GMM_L2_SIZE_DWORD(TTType) (GMM_AUX_L2_SIZE_DWORD)
84 #define GMM_L3_SIZE(TTType) (GMM_AUX_L3_SIZE)
85 #define GMM_L1_ENTRY_IDX(TTType, GfxAddress, pGmmLibContext) (GMM_AUX_L1_ENTRY_IDX((GfxAddress), (pGmmLibContext)))
86 #define GMM_L2_ENTRY_IDX(TTType, GfxAddress) (GMM_AUX_L2_ENTRY_IDX(GfxAddress))
87 #define GMM_L3_ENTRY_IDX(TTType, GfxAddress) (GMM_AUX_L3_ENTRY_IDX(GfxAddress))
88
89 #ifdef GMM_ULT
90 #define GMM_L1_ENTRY_IDX_EXPORTED(TTType, GfxAddress, WA64KEx) GMM_AUX_L1_ENTRY_IDX_EXPORTED((GfxAddress), WA64KEx)
91 #define GMM_L1_ENTRY_IDX_EXPORTED_2(TTType, GfxAddress, WA64KEx, WA16KEx) (GMM_AUX_L1_ENTRY_IDX_EXPORTED_2((GfxAddress), WA64KEx, WA16KEx))
92 #endif
93
94 #ifdef __cplusplus
95 #include "External/Common/GmmMemAllocator.hpp"
96
97 //HW provides single-set of TR/Aux-TT registers for non-privileged programming
98 //Engine-specific offsets are HW-updated with programmed values.
99 #define GET_L3ADROFFSET(TRTT, L3AdrOffset, pGmmLibContext) \
100 L3AdrOffset = 0x4200;
101
102
103 #define ASSIGN_POOLNODE(Pool, NodeIdx, PerTableNodes) { \
104 (Pool)->GetNodeUsageAtIndex((NodeIdx) / (32 *(PerTableNodes))) |= __BIT(((NodeIdx) / (PerTableNodes)) % 32); \
105 (Pool)->GetNodeBBInfoAtIndex(NodeIdx) = SyncInfo(); \
106 (Pool)->GetNumFreeNode() -= (PerTableNodes); \
107 }
108
109 #define DEASSIGN_POOLNODE(PageTableMgr, UmdContext, Pool, NodeIdx, PerTableNodes) { \
110 (Pool)->GetNodeUsageAtIndex((NodeIdx) / (32 * (PerTableNodes))) &= ~__BIT(((NodeIdx) / (PerTableNodes)) % 32 ); \
111 (Pool)->GetNumFreeNode() += (PerTableNodes); \
112 if((Pool)->GetNumFreeNode() == PAGETABLE_POOL_MAX_NODES) { \
113 PageTableMgr->__ReleaseUnusedPool((UmdContext)); \
114 } \
115 }
116
117 namespace GmmLib
118 {
119 #define PAGETABLE_POOL_MAX_NODES 512 //Max. number of L2/L1 tables pool contains
120 #define PAGETABLE_POOL_SIZE_IN_DWORD PAGETABLE_POOL_MAX_NODES / 32
121 #define PAGETABLE_POOL_SIZE PAGETABLE_POOL_MAX_NODES * PAGE_SIZE //Pool for L2/L1 table allocation
122 #define AUX_L2TABLE_SIZE_IN_POOLNODES 8 //Aux L2 is 32KB
123 #define AUX_L1TABLE_SIZE_IN_POOLNODES 2 //Aux L1 is 8KB
124 #define AUX_L1TABLE_SIZE_IN_POOLNODES_2(pGmmLibContext) (pGmmLibContext ? ((WA64K(pGmmLibContext) || WA16K(pGmmLibContext)) ? 2 : 1) : 2) //Aux L1 is 8KB / 4K (MTL)
125 #define PAGETABLE_POOL_MAX_UNUSED_SIZE GMM_MBYTE(16) //Max. size of unused pool, driver keeps resident
126
127
128 //////////////////////////////////////////////////////////////////////////////////////////////
129 /// Contains functions and members for GmmPageTablePool.
130 /// PageTablePool is a Linked-list, provides common location for both Aux TT and TR-TT pages
131 /// Separate NodePool (linked-list element) kept for each PoolType, for cleaner management in
132 /// per-table size
133 /////////////////////////////////////////////////////////////////////////////////////////////
134 class GmmPageTablePool
135 {
136 private:
137 //PageTablePool allocation descriptor
138 GMM_RESOURCE_INFO* pGmmResInfo;
139 HANDLE PoolHandle;
140 GMM_GFX_ADDRESS PoolGfxAddress;
141 GMM_GFX_ADDRESS CPUAddress; //LMEM-cpuvisible adr
142
143 POOL_TYPE PoolType; //Separate Node-pools for TR-L2, TR-L1, Aux-L2, Aux-L1 usages-
144
145 //PageTablePool usage descriptors
146 int NumFreeNodes; //has value {0 to Pool_Max_nodes}
147 uint32_t* NodeUsage; //destined node state (updated during node assignment and removed based on destined state of L1/L2 Table
148 //that used the pool node)
149 //Aux-Pool node-usage tracked at every eighth/second node(for L2 vs L1)
150 //ie 1b per node for TR-table, 1b per 8-nodes for Aux-L2table, 1b per 2-nodes for AuxL1-table
151 //array size= POOL_SIZE_IN_DWORD for TR, =POOL_SIZE_IN_DWORD/8 for AuxL2, POOL_SIZE_IN_DWORD/2 for AuxL1
152
153 SyncInfo* NodeBBInfo; //BB info for pending Gpu usage of each pool node
154 //array of size MaxPoolNodes for TR, =MaxPoolNodes / 8 for Aux, MaxPoolNodes / 2 for AuxL1
155
156 SyncInfo PoolBBInfo; //BB info for Gpu usage of the Pool (most recent of pool node BB info)
157
158 GmmPageTablePool* NextPool; //Next node-Pool in the LinkedList
159 GmmClientContext *pClientContext; ///< ClientContext of the client creating this Object
160 public:
GmmPageTablePool()161 GmmPageTablePool() :
162 pGmmResInfo(NULL),
163 PoolHandle(),
164 PoolGfxAddress(0x0),
165 CPUAddress(0x0),
166 PoolType(POOL_TYPE_TRTTL1),
167 NumFreeNodes(PAGETABLE_POOL_MAX_NODES),
168 NodeUsage(NULL),
169 NodeBBInfo(NULL),
170 PoolBBInfo(),
171 NextPool(NULL),
172 pClientContext(NULL)
173 {
174
175 }
GmmPageTablePool(HANDLE hAlloc,GMM_RESOURCE_INFO * pGmmRes,GMM_GFX_ADDRESS SysMem,POOL_TYPE Type)176 GmmPageTablePool(HANDLE hAlloc, GMM_RESOURCE_INFO* pGmmRes, GMM_GFX_ADDRESS SysMem, POOL_TYPE Type) :
177 GmmPageTablePool()
178 {
179 int DwordPoolSize;
180 GMM_LIB_CONTEXT *pGmmLibContext;
181 PoolHandle = hAlloc;
182 pGmmResInfo = pGmmRes;
183 PoolGfxAddress = SysMem;
184 CPUAddress = PoolGfxAddress;
185 NextPool = NULL;
186 NumFreeNodes = PAGETABLE_POOL_MAX_NODES;
187 PoolType = Type;
188 if(pGmmResInfo)
189 {
190 pClientContext = pGmmResInfo->GetGmmClientContext();
191 }
192 pGmmLibContext = pClientContext ? pClientContext->GetLibContext() : NULL;
193 DwordPoolSize = (Type == POOL_TYPE_AUXTTL1) ? PAGETABLE_POOL_SIZE_IN_DWORD / AUX_L1TABLE_SIZE_IN_POOLNODES_2(pGmmLibContext) : (Type == POOL_TYPE_AUXTTL2) ? PAGETABLE_POOL_SIZE_IN_DWORD / AUX_L2TABLE_SIZE_IN_POOLNODES : PAGETABLE_POOL_SIZE_IN_DWORD;
194 NodeUsage = new uint32_t[DwordPoolSize]();
195 NodeBBInfo = new SyncInfo[DwordPoolSize * 32]();
196 }
GmmPageTablePool(HANDLE hAlloc,GMM_RESOURCE_INFO * pGmmRes,GMM_GFX_ADDRESS GfxAdr,GMM_GFX_ADDRESS CPUAdr,POOL_TYPE Type)197 GmmPageTablePool(HANDLE hAlloc, GMM_RESOURCE_INFO* pGmmRes, GMM_GFX_ADDRESS GfxAdr, GMM_GFX_ADDRESS CPUAdr, POOL_TYPE Type) :
198 GmmPageTablePool(hAlloc, pGmmRes, GfxAdr, Type)
199 {
200 CPUAddress = (CPUAdr != GfxAdr) ? CPUAdr : GfxAdr;
201 }
~GmmPageTablePool()202 ~GmmPageTablePool()
203 {
204 delete[] NodeUsage;
205 delete[] NodeBBInfo;
206 }
207
InsertInList(GmmPageTablePool * NewNode)208 GmmPageTablePool* InsertInList(GmmPageTablePool* NewNode)
209 {
210 GmmPageTablePool *Node = this;
211 while (Node->NextPool)
212 {
213 Node = Node->NextPool;
214 }
215 Node->NextPool = NewNode;
216 return Node->NextPool;
217 }
218
InsertInListAtBegin(GmmPageTablePool * NewNode)219 GmmPageTablePool* InsertInListAtBegin(GmmPageTablePool* NewNode)
220 {
221 GmmPageTablePool *Node = this;
222 NewNode->NextPool = Node;
223 return NewNode;
224 }
225
GetNextPool()226 GmmPageTablePool* &GetNextPool() { return NextPool; }
GetPoolHandle()227 HANDLE& GetPoolHandle() { return PoolHandle; }
GetPoolType()228 POOL_TYPE& GetPoolType() { return PoolType; }
GetNumFreeNode()229 int& GetNumFreeNode() { return NumFreeNodes; }
GetPoolBBInfo()230 SyncInfo& GetPoolBBInfo() { return PoolBBInfo; }
GetNodeUsageAtIndex(int j)231 uint32_t& GetNodeUsageAtIndex(int j) { return NodeUsage[j]; }
GetNodeBBInfoAtIndex(int j)232 SyncInfo& GetNodeBBInfoAtIndex(int j)
233 {
234 GMM_LIB_CONTEXT *pGmmLibContext = pClientContext ? pClientContext->GetLibContext() : NULL;
235 int BBInfoNodeIdx = (PoolType == POOL_TYPE_AUXTTL1) ? j / AUX_L1TABLE_SIZE_IN_POOLNODES_2(pGmmLibContext) : (PoolType == POOL_TYPE_AUXTTL2) ? j / AUX_L2TABLE_SIZE_IN_POOLNODES : j;
236 return NodeBBInfo[BBInfoNodeIdx];
237 }
GetGfxAddress()238 GMM_GFX_ADDRESS GetGfxAddress() { return PoolGfxAddress; }
GetCPUAddress()239 GMM_GFX_ADDRESS GetCPUAddress() { return CPUAddress; }
GetGmmResInfo()240 GMM_RESOURCE_INFO* &GetGmmResInfo() { return pGmmResInfo; }
IsPoolInUse(SyncInfo BBInfo)241 bool IsPoolInUse(SyncInfo BBInfo) {
242 if (NumFreeNodes < PAGETABLE_POOL_MAX_NODES ||
243 (PoolBBInfo.BBQueueHandle == BBInfo.BBQueueHandle &&
244 PoolBBInfo.BBFence == BBInfo.BBFence + 1)) //Pool will be used by next BB submission, freeing it will cause page fault
245 {
246 return true;
247 }
248 return false;
249 }
250 bool __IsUnusedTRTTPoolOverLimit(GMM_GFX_SIZE_T * OverLimitSize);
251 void ClearBBReference(void * BBQHandle);
252 GMM_STATUS __DestroyPageTablePool(void * DeviceCallbacks,HANDLE hCsr);
253 };
254
255 //////////////////////////////////////////////////////////////////////////////////////////////
256 /// Contains functions and members for Table.
257 /// Table defines basic building block for tables at different page-table levels
258 /////////////////////////////////////////////////////////////////////////////////////////////
259 class Table
260 {
261 protected:
262 GMM_PAGETABLEPool *PoolElem; //L2 Pool ptr different for L2Tables when Pool_nodes <512
263 int PoolNodeIdx; //pool node idx used for L2 Table
264 SyncInfo BBInfo; //BB Handle/fence using Table
265 uint32_t * UsedEntries; //Tracks which L1/L2 entries are being used
266 //array size GMM_L1_SIZE_DWORD(TT-type) for LastLevelTable, MidLeveltable(??)
267 //array of 1024/32=32 DWs for TR-table, 4096/32 =512 for Aux-Table
268 public:
Table()269 Table() :
270 PoolElem(NULL),
271 PoolNodeIdx(),
272 BBInfo(),
273 UsedEntries(NULL)
274 {
275 }
GetNodeIdx()276 int& GetNodeIdx() { return PoolNodeIdx; }
GetPool()277 GmmPageTablePool* &GetPool() { return PoolElem; }
GetCPUAddress()278 GMM_GFX_ADDRESS GetCPUAddress() { return (PoolElem->GetCPUAddress() + (PoolNodeIdx * PAGE_SIZE)); }
GetBBInfo()279 SyncInfo& GetBBInfo() { return BBInfo; }
GetUsedEntries()280 uint32_t* &GetUsedEntries() { return UsedEntries; }
281 bool TrackTableUsage(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr, bool NullMapped,GMM_LIB_CONTEXT* pGmmLibContext);
282 bool IsTableNullMapped(TT_TYPE Type, bool IsL1, GMM_GFX_ADDRESS TileAdr,GMM_LIB_CONTEXT *pGmmLibContext);
283 void UpdatePoolFence(GMM_UMD_SYNCCONTEXT * UmdContext, bool ClearNode);
284 };
285
286 //////////////////////////////////////////////////////////////////////////////////////////////
287 /// Contains functions and members for LastLevelTable.
288 /// LastLevelTable defines leaf level tables in multi-level pageTable structure
289 /////////////////////////////////////////////////////////////////////////////////////////////
290 class LastLevelTable : public Table
291 {
292 private:
293 uint32_t L2eIdx;
294 LastLevelTable *pNext;
295
296 public:
LastLevelTable()297 LastLevelTable() : Table(),
298 L2eIdx() //Pass in Aux vs TR table's GMM_L2_SIZE and initialize L2eIdx?
299 {
300 pNext = NULL;
301 }
302
LastLevelTable(GMM_PAGETABLEPool * Elem,int NodeIdx,int DwordL1e,int L2eIndex)303 LastLevelTable(GMM_PAGETABLEPool *Elem, int NodeIdx, int DwordL1e, int L2eIndex)
304 : LastLevelTable()
305 {
306 PoolElem = Elem;
307 PoolNodeIdx = NodeIdx;
308 BBInfo = Elem->GetNodeBBInfoAtIndex(NodeIdx);
309 L2eIdx = L2eIndex;
310 pNext = NULL;
311 UsedEntries = new uint32_t[DwordL1e]();
312 }
~LastLevelTable()313 ~LastLevelTable()
314 {
315 delete[] UsedEntries;
316 }
317
GetL2eIdx()318 int GetL2eIdx() {
319 return L2eIdx;
320 }
321
Next()322 LastLevelTable* &Next() {
323 return pNext;
324 }
325 };
326
327 //////////////////////////////////////////////////////////////////////////////////////////////
328 /// Contains functions and members for MidLevelTable.
329 /// MidLevelTable defines secondary level tables in multi-level pageTable structure
330 /////////////////////////////////////////////////////////////////////////////////////////////
331 class MidLevelTable : public Table
332 {
333 private:
334 LastLevelTable *pTTL1; //linked list of L1 tables
335
336 public:
MidLevelTable()337 MidLevelTable() :Table()
338 {
339 pTTL1 = NULL;
340 }
MidLevelTable(GMM_PAGETABLEPool * Pool,int NodeIdx,SyncInfo Info)341 MidLevelTable(GMM_PAGETABLEPool *Pool, int NodeIdx, SyncInfo Info) : MidLevelTable()
342 {
343 PoolElem = Pool;
344 BBInfo = Info;
345 PoolNodeIdx = NodeIdx;
346 }
~MidLevelTable()347 ~MidLevelTable()
348 {
349 if (pTTL1)
350 {
351 LastLevelTable* item = pTTL1;
352
353 while (item)
354 {
355 LastLevelTable* nextItem = item->Next();
356 delete item;
357 item = nextItem;
358 }
359
360 pTTL1 = NULL;
361 }
362 }
363 LastLevelTable* GetL1Table(GMM_GFX_SIZE_T L2eIdx, LastLevelTable** Prev = NULL)
364 {
365 LastLevelTable* pL1Tbl = pTTL1;
366 LastLevelTable* PrevL1Tbl = NULL;
367
368 while (pL1Tbl)
369 {
370 if (pL1Tbl->GetL2eIdx() == L2eIdx)
371 {
372 break;
373 }
374 PrevL1Tbl = pL1Tbl;
375 pL1Tbl = pL1Tbl->Next();
376 }
377
378 //if requested, save previous node in linked-list
379 if (Prev)
380 {
381 *Prev = PrevL1Tbl;
382 }
383 return pL1Tbl;
384 }
InsertInList(LastLevelTable * pL1Tbl)385 void InsertInList(LastLevelTable* pL1Tbl)
386 {
387 LastLevelTable* Prev = pTTL1;
388
389 //Insert at end
390 while (Prev && Prev->Next())
391 {
392 Prev = Prev->Next();
393 }
394
395 if (Prev)
396 {
397 Prev->Next() = pL1Tbl;
398 }
399 else
400 {
401 pTTL1 = pL1Tbl;
402 }
403 }
DeleteFromList(LastLevelTable * pL1Tbl,LastLevelTable * PrevL1Tbl)404 void DeleteFromList(LastLevelTable* pL1Tbl, LastLevelTable* PrevL1Tbl)
405 {
406 //Save next L1Table in list, before deleting current one
407 if (pL1Tbl)
408 {
409 if (PrevL1Tbl)
410 {
411 PrevL1Tbl->Next() = pL1Tbl->Next();
412 }
413 else
414 {
415 pTTL1 = pL1Tbl->Next();
416 }
417
418 delete pL1Tbl;
419 }
420 }
421 };
422
423 /////////////////////////////////////////////////////
424 /// Contains functions and members for PageTable.
425 /// PageTable defines multi-level pageTable
426 /////////////////////////////////////////////////////
427 class PageTable :
428 public GmmMemAllocator
429 {
430 protected:
431 const TT_TYPE TTType; //PageTable is AuxTT
432 const int NodesPerTable; //Aux L2/L3 has 32KB size, Aux L1 has 4KB -can't use as selector for PageTable is AuxTT
433 // 1 node for TR-table, 8 nodes for Aux-Table L2, 2 nodes for Aux-table L1
434 //Root Table structure
435 struct RootTable
436 {
437 GMM_RESOURCE_INFO* pGmmResInfo;
438 HANDLE L3Handle;
439 GMM_GFX_ADDRESS GfxAddress; //L3 Table Adr CPU equivalent GPU addr
440 GMM_GFX_ADDRESS CPUAddress; //LMEM-cpuvisible adr
441 bool NeedRegisterUpdate; //True @ L3 allocation, False when L3AdrRegWrite done
442 SyncInfo BBInfo;
RootTableRootTable443 RootTable() : pGmmResInfo(NULL), L3Handle(NULL), GfxAddress(0), CPUAddress(0), NeedRegisterUpdate(false), BBInfo() {}
444 } TTL3;
445
446 MidLevelTable* pTTL2; //array of L2-Tables
447
448 public:
449 #ifdef _WIN32
450 CRITICAL_SECTION TTLock; //synchronized access of PageTable obj
451 #elif defined __linux__
452 pthread_mutex_t TTLock;
453 #endif
454
455 GmmPageTableMgr* PageTableMgr;
456 GmmClientContext *pClientContext;
457
PageTable(int Size,int NumL3e,TT_TYPE flag)458 PageTable(int Size, int NumL3e, TT_TYPE flag) :
459 TTType(flag),
460 NodesPerTable(Size / PAGE_SIZE)
461 {
462 PageTableMgr = NULL;
463 pClientContext = NULL;
464 InitializeCriticalSection(&TTLock);
465
466 pTTL2 = new MidLevelTable[NumL3e];
467 }
468
~PageTable()469 ~PageTable()
470 {
471 delete[] pTTL2;
472
473 DeleteCriticalSection(&TTLock);
474 }
475
GetGmmLibContext()476 inline GMM_LIB_CONTEXT* GetGmmLibContext()
477 {
478 return pClientContext->GetLibContext();
479 }
480
GetL3Address()481 GMM_GFX_ADDRESS GetL3Address() { return TTL3.GfxAddress; }
GetRegisterStatus()482 bool &GetRegisterStatus() { return TTL3.NeedRegisterUpdate; }
483 GMM_STATUS AllocateL3Table(uint32_t L3TableSize, uint32_t L3AddrAlignment);
484 GMM_STATUS DestroyL3Table();
485 void AllocateL1L2Table(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS * L1TableAdr, GMM_GFX_ADDRESS * L2TableAdr);
486 void AllocateDummyTables(GmmLib::Table **L2Table, GmmLib::Table **L1Table);
487 void GetL1L2TableAddr(GMM_GFX_ADDRESS TileAddr, GMM_GFX_ADDRESS * L1TableAdr, GMM_GFX_ADDRESS* L2TableAdr);
488 uint8_t GetMappingType(GMM_GFX_ADDRESS GfxVA, GMM_GFX_SIZE_T Size, GMM_GFX_ADDRESS& LastAddr);
GetL3Handle()489 HANDLE GetL3Handle() { return TTL3.L3Handle; }
490 };
491
492 //////////////////////////////////////////////////////////////////////////////////////////////
493 /// Contains functions and members for AuxTable.
494 /// AuxTable defines PageTable for translating VA->AuxVA, ie defines page-walk to get address
495 /// of CCS-cacheline containing auxiliary data (compression tag, etc) for some resource
496 /////////////////////////////////////////////////////////////////////////////////////////////
497 class AuxTable : public PageTable
498 {
499 public:
500 const int L1Size;
501 Table* NullL2Table;
502 Table* NullL1Table;
503 GMM_GFX_ADDRESS NullCCSTile;
AuxTable(GmmClientContext * pClientContextIn)504 AuxTable(GmmClientContext *pClientContextIn)
505 : PageTable(8 * PAGE_SIZE, GMM_AUX_L3_SIZE, TT_TYPE::AUXTT), L1Size((WA16K(pClientContextIn->GetLibContext()) || WA64K(pClientContextIn->GetLibContext())) ? (2 * PAGE_SIZE) : PAGE_SIZE)
506 {
507 NullL2Table = nullptr;
508 NullL1Table = nullptr;
509 NullCCSTile = 0;
510 }
AuxTable()511 AuxTable()
512 : PageTable(8 * PAGE_SIZE, GMM_AUX_L3_SIZE, TT_TYPE::AUXTT), L1Size(2 * PAGE_SIZE)
513 {
514 NullL2Table = nullptr;
515 NullL1Table = nullptr;
516 NullCCSTile = 0;
517 }
518 GMM_STATUS InvalidateTable(GMM_UMD_SYNCCONTEXT * UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint8_t DoNotWait);
519
520 GMM_STATUS MapValidEntry(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T BaseSize,
521 GMM_RESOURCE_INFO* BaseResInfo, GMM_GFX_ADDRESS AuxVA, GMM_RESOURCE_INFO* AuxResInfo, uint64_t PartialData, uint8_t DoNotWait);
522
523 GMM_STATUS MapNullCCS(GMM_UMD_SYNCCONTEXT *UmdContext, GMM_GFX_ADDRESS BaseAdr, GMM_GFX_SIZE_T Size, uint64_t PartialL1e, uint8_t DoNotWait);
524
525 GMM_AUXTTL1e CreateAuxL1Data(GMM_RESOURCE_INFO* BaseResInfo);
526 GMM_GFX_ADDRESS GMM_INLINE __GetCCSCacheline(GMM_RESOURCE_INFO* BaseResInfo, GMM_GFX_ADDRESS BaseAdr, GMM_RESOURCE_INFO* AuxResInfo,
527 GMM_GFX_ADDRESS AuxVA, GMM_GFX_SIZE_T AdrOffset);
528
529 };
530
531 typedef struct _GMM_DEVICE_ALLOC {
532 uint32_t Size;
533 uint32_t Alignment;
534 HANDLE Handle;
535 GMM_GFX_ADDRESS GfxVA;
536 GMM_GFX_ADDRESS CPUVA;
537 void * Priv;
538 HANDLE hCsr;
539 } GMM_DEVICE_ALLOC;
540
541 typedef struct _GMM_DEVICE_DEALLOC {
542 HANDLE Handle;
543 GMM_GFX_ADDRESS GfxVA;
544 void * Priv;
545 HANDLE hCsr;
546 } GMM_DEVICE_DEALLOC;
547
548 GMM_STATUS __GmmDeviceAlloc(GmmClientContext *pClientContext,
549 GMM_DEVICE_CALLBACKS_INT *pDeviceCbInt,
550 GMM_DEVICE_ALLOC *pAlloc);
551
552 GMM_STATUS __GmmDeviceDealloc(GMM_CLIENT ClientType,
553 GMM_DEVICE_CALLBACKS_INT *DeviceCb,
554 GMM_DEVICE_DEALLOC *pDealloc,
555 GmmClientContext *pClientContext);
556 }
557 #endif // #ifdef __cplusplus
558
559