1 /*==============================================================================
2 Copyright(c) 2019 Intel Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a
5 copy of this software and associated documentation files(the "Software"),
6 to deal in the Software without restriction, including without limitation
7 the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 and / or sell copies of the Software, and to permit persons to whom the
9 Software is furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included
12 in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 OTHER DEALINGS IN THE SOFTWARE.
21
22 Description: UMD-TT manager (manages both TR-TT and AUX-TT in user mode space)
23
24 ============================================================================*/
25
26 #include "Internal/Common/GmmLibInc.h"
27 #include "External/Common/GmmPageTableMgr.h"
28 #include "../TranslationTable/GmmUmdTranslationTable.h"
29 #include "External/Common/GmmClientContext.h"
30
31 #if defined(__linux__)
32 #include "Internal/Linux/GmmResourceInfoLinInt.h"
33 #endif
34
35 #define ENTER_CRITICAL_SECTION \
36 if(AuxTTObj) \
37 { \
38 EnterCriticalSection(&PoolLock); \
39 }
40
41 #define EXIT_CRITICAL_SECTION \
42 if(AuxTTObj) \
43 { \
44 LeaveCriticalSection(&PoolLock); \
45 }
46 extern GMM_MA_LIB_CONTEXT *pGmmMALibContext;
47 #if defined(__linux__)
__GmmDeviceAlloc(GmmClientContext * pClientContext,GMM_DEVICE_CALLBACKS_INT * pDeviceCbInt,GMM_DEVICE_ALLOC * pAlloc)48 GMM_STATUS GmmLib::__GmmDeviceAlloc(GmmClientContext * pClientContext,
49 GMM_DEVICE_CALLBACKS_INT *pDeviceCbInt,
50 GMM_DEVICE_ALLOC * pAlloc)
51 {
52 GMM_CLIENT ClientType;
53 GMM_DDI_ALLOCATE Alloc = {0};
54 int err;
55
56 GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
57
58 __GMM_ASSERTPTR(GmmCheckForNullDevCbPfn(ClientType, pDeviceCbInt, GMM_DEV_CB_ALLOC), GMM_INVALIDPARAM);
59
60 if(GmmCheckForNullDevCbPfn(ClientType, pDeviceCbInt, GMM_DEV_CB_ALLOC))
61 {
62 Alloc.size = pAlloc->Size;
63 Alloc.alignment = pAlloc->Alignment;
64
65 err = GmmDeviceCallback(ClientType, pDeviceCbInt, &Alloc);
66 if(err)
67 {
68 return GMM_OUT_OF_MEMORY;
69 }
70
71 pAlloc->GfxVA = Alloc.gfxAddr;
72 pAlloc->CPUVA = (GMM_GFX_ADDRESS) Alloc.cpuAddr;
73 pAlloc->Handle = (HANDLE)Alloc.bo;
74 }
75
76 return GMM_SUCCESS;
77 }
78
__GmmDeviceDealloc(GMM_CLIENT ClientType,GMM_DEVICE_CALLBACKS_INT * DeviceCb,GMM_DEVICE_DEALLOC * pDealloc,GmmClientContext * pClientContext)79 GMM_STATUS GmmLib::__GmmDeviceDealloc(GMM_CLIENT ClientType,
80 GMM_DEVICE_CALLBACKS_INT *DeviceCb,
81 GMM_DEVICE_DEALLOC * pDealloc,
82 GmmClientContext * pClientContext)
83
84 {
85 GMM_DDI_DEALLOCATE DeAlloc = {0};
86 int err = 0;
87
88 __GMM_ASSERTPTR(GmmCheckForNullDevCbPfn(ClientType, DeviceCb, GMM_DEV_CB_DEALLOC), GMM_INVALIDPARAM);
89
90 if(GmmCheckForNullDevCbPfn(ClientType, DeviceCb, GMM_DEV_CB_DEALLOC))
91 {
92 DeAlloc.bo = pDealloc->Handle;
93
94 err = GmmDeviceCallback(ClientType, DeviceCb, &DeAlloc);
95 }
96
97 return (err == 0) ? GMM_SUCCESS : GMM_ERROR;
98 }
99 #endif
100
101 //=============================================================================
102 //
103 // Function: __AllocateNodePool
104 //
105 // Desc: Allocates (always resident SVM) memory for new Pool node, and updates PageTableMgr object
106 //
107 // Parameters:
108 // AddrAlignment: Pool allocation address alignment
109 //
110 // Returns:
111 // S_OK on success,
112 //-----------------------------------------------------------------------------
__AllocateNodePool(uint32_t AddrAlignment,GmmLib::POOL_TYPE Type)113 GmmLib::GMM_PAGETABLEPool *GmmLib::GmmPageTableMgr::__AllocateNodePool(uint32_t AddrAlignment, GmmLib::POOL_TYPE Type)
114 {
115 GMM_STATUS Status = GMM_SUCCESS;
116 GMM_RESOURCE_INFO *pGmmResInfo = NULL;
117 GMM_PAGETABLEPool *pTTPool = NULL;
118 HANDLE PoolHnd = 0;
119 GMM_CLIENT ClientType;
120 GMM_DEVICE_ALLOC Alloc = {0};
121
122 ENTER_CRITICAL_SECTION
123
124 //Allocate pool, sized PAGETABLE_POOL_MAX_NODES pages, assignable to TR/Aux L1/L2 tables
125 //SVM allocation, always resident
126 Alloc.Size = PAGETABLE_POOL_SIZE;
127 Alloc.Alignment = AddrAlignment;
128 Alloc.hCsr = hCsr;
129
130 Status = __GmmDeviceAlloc(pClientContext, &DeviceCbInt, &Alloc);
131
132 if(Status != GMM_SUCCESS)
133 {
134 __GMM_ASSERT(0);
135 EXIT_CRITICAL_SECTION
136 return NULL;
137 }
138
139 PoolHnd = Alloc.Handle;
140 pGmmResInfo = (GMM_RESOURCE_INFO *)Alloc.Priv;
141
142 pTTPool = new GMM_PAGETABLEPool(PoolHnd, pGmmResInfo, Alloc.GfxVA, Alloc.CPUVA, Type);
143
144
145 if(pTTPool)
146 {
147 if(pPool)
148 {
149 NumNodePoolElements++;
150 if(Type == POOL_TYPE_TRTTL2) // TRTT-L2 not 1st node in Pool LinkedList, place it at beginning
151 {
152 pPool = pPool->InsertInListAtBegin(pTTPool);
153 }
154 else
155 {
156 pTTPool = pPool->InsertInList(pTTPool);
157 }
158 }
159 else
160 {
161 NumNodePoolElements = 1;
162 pPool = pTTPool;
163 }
164 }
165 else
166 {
167 __GMM_ASSERT(0);
168 Status = GMM_OUT_OF_MEMORY;
169 }
170
171 EXIT_CRITICAL_SECTION
172 return (Status == GMM_SUCCESS) ? pTTPool : NULL;
173 }
174
175 //=============================================================================
176 //
177 // Function: __ReleaseUnusedPool
178 //
179 // Desc: Frees up unused PageTablePools once residency limit is hit
180 //
181 // Parameters:
182 // UmdContext: pointer to caller thread's context (containing BBHandle/Fence info)
183 //
184 //-----------------------------------------------------------------------------
__ReleaseUnusedPool(GMM_UMD_SYNCCONTEXT * UmdContext)185 void GmmLib::GmmPageTableMgr::__ReleaseUnusedPool(GMM_UMD_SYNCCONTEXT *UmdContext)
186 {
187 GMM_STATUS Status = GMM_SUCCESS;
188 GMM_GFX_SIZE_T PoolSizeToFree = {0};
189 GMM_GFX_SIZE_T FreedSize = {0};
190 GmmLib::GMM_PAGETABLEPool *Pool = NULL, *PrevPool = NULL;
191 uint32_t i = 0;
192 GMM_CLIENT ClientType;
193 GMM_DEVICE_DEALLOC Dealloc;
194
195 GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
196
197 ENTER_CRITICAL_SECTION
198 if(pPool->__IsUnusedTRTTPoolOverLimit(&PoolSizeToFree))
199 {
200 for(i = 0; i < NumNodePoolElements && FreedSize < PoolSizeToFree; i++)
201 {
202 Pool = (PrevPool) ? PrevPool->GetNextPool() : pPool;
203
204 if(Pool->IsPoolInUse(UmdContext ? SyncInfo(UmdContext->BBFenceObj, UmdContext->BBLastFence) : SyncInfo()))
205 {
206 PrevPool = Pool;
207 continue;
208 }
209
210 if(GmmCheckForNullDevCbPfn(ClientType, &DeviceCbInt, GMM_DEV_CB_WAIT_FROM_CPU))
211 {
212 GMM_DDI_WAITFORSYNCHRONIZATIONOBJECTFROMCPU Wait = {0};
213 Wait.bo = Pool->GetPoolHandle();
214 GmmDeviceCallback(ClientType, &DeviceCbInt, &Wait);
215 }
216
217 Dealloc.Handle = Pool->GetPoolHandle();
218 Dealloc.GfxVA = Pool->GetGfxAddress();
219 Dealloc.Priv = Pool->GetGmmResInfo();
220 Dealloc.hCsr = hCsr;
221
222 Status = __GmmDeviceDealloc(ClientType, &DeviceCbInt, &Dealloc, pClientContext);
223
224 __GMM_ASSERT(GMM_SUCCESS == Status);
225
226 if(PrevPool)
227 {
228 PrevPool->GetNextPool() = Pool->GetNextPool();
229 }
230 else
231 {
232 pPool = Pool->GetNextPool();
233 }
234 delete Pool;
235 FreedSize += PAGETABLE_POOL_SIZE;
236 }
237 }
238 EXIT_CRITICAL_SECTION
239 }
240
241 //=============================================================================
242 //
243 // Function: __GetFreePoolNode
244 //
245 // Desc: Finds free node within existing PageTablePool(s), if no such node found,
246 // allocates new PageTablePool. Caller should update Pool Node usage
247 //
248 // Parameters:
249 // FreePoolNodeIdx: pointer to return Pool's free Node index
250 // PoolType: AuxTT_L1/L2 pool
251 //
252 // Returns:
253 // PageTablePool element and FreePoolNodeIdx that should be used for L2/L1 assignment
254 // NULL, if no free node exists and new pool allocation failed
255 //-----------------------------------------------------------------------------
__GetFreePoolNode(uint32_t * FreePoolNodeIdx,POOL_TYPE PoolType)256 GmmLib::GMM_PAGETABLEPool *GmmLib::GmmPageTableMgr::__GetFreePoolNode(uint32_t *FreePoolNodeIdx, POOL_TYPE PoolType)
257 {
258 uint32_t PoolNode = -1, i = 0, j = 0, DWdivisor = 1, IdxMultiplier = 1;
259 bool PoolNodeFound = false, TRTTPool = false;
260
261 ENTER_CRITICAL_SECTION
262 GmmLib::GMM_PAGETABLEPool *Pool = pPool;
263
264 Pool = (PoolType == POOL_TYPE_TRTTL2) ? Pool : //1st pool reserved for TRTT-L2, since TRTT-L2 pruning not supported yet,
265 (Pool ? Pool->GetNextPool() : NULL); //other pools can be TR-L1/Aux-L1/Aux-L2 (and support dynamic pruning)
266 TRTTPool = (PoolType == POOL_TYPE_TRTTL2 || PoolType == POOL_TYPE_TRTTL1) ? true : false;
267 DWdivisor = TRTTPool ? 8 * sizeof(uint32_t) : (PoolType == POOL_TYPE_AUXTTL2) ? 8 * sizeof(uint32_t) * AUX_L2TABLE_SIZE_IN_POOLNODES : 8 * sizeof(uint32_t) * AUX_L1TABLE_SIZE_IN_POOLNODES_2(GetLibContext());
268 IdxMultiplier = TRTTPool ? 1 : (PoolType == POOL_TYPE_AUXTTL2) ? AUX_L2TABLE_SIZE_IN_POOLNODES : AUX_L1TABLE_SIZE_IN_POOLNODES_2(GetLibContext());
269 //Scan existing PageTablePools for free pool node
270 for(i = (PoolType == POOL_TYPE_TRTTL2) ? 0 : 1; Pool && i < NumNodePoolElements; i++)
271 {
272 if(Pool->GetNumFreeNode() > 0 && Pool->GetPoolType() == PoolType)
273 {
274 PoolNodeFound = true;
275 *FreePoolNodeIdx = 0;
276 for(; j < PAGETABLE_POOL_MAX_NODES / DWdivisor; j++)
277 {
278 if(_BitScanForward((uint32_t *)&PoolNode, (uint32_t) ~(Pool->GetNodeUsageAtIndex(j)))) // Get LSB that has value 0
279 {
280 *FreePoolNodeIdx += PoolNode * IdxMultiplier;
281 PoolNodeFound = true;
282 break;
283 }
284 PoolNodeFound = false;
285 *FreePoolNodeIdx += DWdivisor; //DWORD size in bits
286 }
287 }
288 if(PoolNodeFound)
289 {
290 __GMM_ASSERT(Pool->GetPoolType() == PoolType);
291 EXIT_CRITICAL_SECTION
292 return Pool;
293 }
294 Pool = Pool->GetNextPool();
295 }
296
297 //No free pool node, allocate new
298 if(!PoolNodeFound)
299 {
300 GMM_PAGETABLEPool *Pool = NULL;
301 if(Pool = __AllocateNodePool(IdxMultiplier * PAGE_SIZE, PoolType))
302 {
303 __GMM_ASSERT(Pool->GetPoolType() == PoolType);
304
305 *FreePoolNodeIdx = 0;
306 EXIT_CRITICAL_SECTION
307 return Pool;
308 }
309 }
310
311 EXIT_CRITICAL_SECTION
312 return NULL;
313 }
314
315
316
317 /**********************************************************************************
318 ** Class GmmPageTableMgr functions **
319 ***********************************************************************************/
320
321 /////////////////////////////////////////////////////////////////////////////////////
322 /// Instantiates GmmPageTableMgr, allocating memory for root-tables, copies provided
323 /// device-callback function pointers
324 ///
325 /// @param[in] DeviceCb: pointer sharing device-callback function pointers
326 /// @param[in] TTFlags: Flags specifying which PageTables are required by client
327 /// @return GmmPageTableMgr*
328 /////////////////////////////////////////////////////////////////////////////////////
GmmPageTableMgr(GMM_DEVICE_CALLBACKS_INT * DeviceCB,uint32_t TTFlags,GmmClientContext * pClientContextIn)329 GmmLib::GmmPageTableMgr::GmmPageTableMgr(GMM_DEVICE_CALLBACKS_INT *DeviceCB, uint32_t TTFlags, GmmClientContext *pClientContextIn)
330 : GmmPageTableMgr()
331 {
332 GMM_PAGETABLE_MGR *ptr = NULL;
333 GMM_STATUS status = GMM_SUCCESS;
334 GMM_CLIENT ClientType;
335
336 if(pClientContextIn)
337 {
338 ClientType = pClientContextIn->GetClientType();
339 }
340 else
341 {
342 goto ERROR_CASE;
343 }
344
345 // this is needed if there is an error case and destructor gets called on ptr
346 this->pClientContext = pClientContextIn;
347
348 // Currently coping the code below to GMMOldAPi.cpp for backward compatible.
349 // Any changes here should be copied there.
350 //Initialize PageTableMgr further, only if PageTable creation succeeded
351 try
352 {
353 ptr = new GmmPageTableMgr();
354 ptr->pClientContext = pClientContextIn;
355 memcpy(&ptr->DeviceCbInt, DeviceCB, sizeof(GMM_DEVICE_CALLBACKS_INT));
356
357 if(pClientContextIn->GetSkuTable().FtrE2ECompression &&
358 !pClientContextIn->GetSkuTable().FtrFlatPhysCCS)
359 {
360 __GMM_ASSERT(TTFlags & AUXTT); //Aux-TT is mandatory
361 ptr->AuxTTObj = new AuxTable(pClientContext);
362 if(!ptr->AuxTTObj)
363 {
364 goto ERROR_CASE;
365 }
366 ptr->AuxTTObj->PageTableMgr = ptr;
367 ptr->AuxTTObj->pClientContext = pClientContextIn;
368 status = ptr->AuxTTObj->AllocateL3Table(8 * PAGE_SIZE, 8 * PAGE_SIZE);
369
370 if(status != GMM_SUCCESS)
371 {
372 InitializeCriticalSection(&(ptr->PoolLock));
373 goto ERROR_CASE;
374 }
375 }
376 }
377
378 catch(...)
379 {
380 __GMM_ASSERT(false);
381 if(ptr && (AuxTTObj))
382 {
383 InitializeCriticalSection(&(ptr->PoolLock));
384 }
385 goto ERROR_CASE;
386 }
387
388 if(status == GMM_SUCCESS && !(AuxTTObj))
389 {
390 if(ptr->AuxTTObj)
391 {
392 ptr->AuxTTObj->PageTableMgr = this;
393 }
394 *this = *ptr;
395 //Don't initialize PoolLock until any of AuxTable object created
396 if(ptr->AuxTTObj )
397 {
398 InitializeCriticalSection(&PoolLock);
399
400 }
401 //Delete temporary ptr, but don't release allocated PageTable Obj.
402 ptr->AuxTTObj = NULL;
403 }
404
405 ERROR_CASE:
406 delete ptr;
407 ptr = NULL;
408 }
409
410 /////////////////////////////////////////////////////////////////////////////////////
411 /// Returns Root-table address for Aux-table
412 ///
413 /// @return GMM_GFX_ADDRESS if Aux-Table was created; NULL otherwise
414 /////////////////////////////////////////////////////////////////////////////////////
GetAuxL3TableAddr()415 GMM_GFX_ADDRESS GmmLib::GmmPageTableMgr::GetAuxL3TableAddr()
416 {
417 return AuxTTObj ? AuxTTObj->GetL3Address() : 0ULL;
418 }
419
420 /////////////////////////////////////////////////////////////////////////////////////
421 /// Queues commands to initialize Aux-Table registers in the HW context image
422 ///
423 /// @param[in] initialBBHandle: pointer to BatchBuffer for queuing commands
424 /// @param[in] engType: specifes engine on which the context would run
425 /// @return GMM_SUCCESS if queuing succeeded; GMM_ERROR otherwise
426 /////////////////////////////////////////////////////////////////////////////////////
InitContextAuxTableRegister(HANDLE CmdQHandle,GMM_ENGINE_TYPE engType)427 GMM_STATUS GmmLib::GmmPageTableMgr::InitContextAuxTableRegister(HANDLE CmdQHandle, GMM_ENGINE_TYPE engType)
428 {
429 GMM_GFX_ADDRESS MaskedL3GfxAddress = 0ULL;
430 GMM_UNREFERENCED_PARAMETER(engType);
431
432 //Check FtrE2ECompression = 1
433 if(GetLibContext()->GetSkuTable().FtrE2ECompression && AuxTTObj != NULL)
434 {
435 EnterCriticalSection(&AuxTTObj->TTLock);
436 if(CmdQHandle)
437 {
438 //engType = ENGINE_TYPE_RCS; //use correct offset based on engType (once per-eng offsets known)
439 uint64_t RegOffset = 0, L3AdrReg = 0;
440 GET_L3ADROFFSET(0, L3AdrReg, GetLibContext());
441
442 RegOffset = (L3AdrReg + sizeof(uint32_t));
443 RegOffset = L3AdrReg | (RegOffset << 0x20);
444
445 MaskedL3GfxAddress = AuxTTObj->GetL3Address();
446
447 //TTCb.pfPrologTranslationTable(CmdQHandle); //MI_FLUSH, TLBInv not required since its called during context-init
448
449 TTCb.pfWriteL3Adr(CmdQHandle, MaskedL3GfxAddress, RegOffset);
450
451 GMM_DPF(GFXDBG_NORMAL, "AuxTT Map Address: GPUVA=0x%016llX\n", MaskedL3GfxAddress);
452
453 //TTCb.pfEpilogTranslationTable(CmdQHandle, 0);
454
455 AuxTTObj->GetRegisterStatus() = 0;
456 }
457 else
458 {
459 __GMM_ASSERT(false);
460 LeaveCriticalSection(&AuxTTObj->TTLock);
461 return GMM_INVALIDPARAM;
462 }
463 LeaveCriticalSection(&AuxTTObj->TTLock);
464 }
465 return GMM_SUCCESS;
466 }
467
468 /////////////////////////////////////////////////////////////////////////////////////
469 /// Updates the Aux-PageTables, for given base resource, with appropriate mappings
470 ///
471 /// @param[in] Details of AuxTable update request
472 /// @return GMM_STATUS
473 /////////////////////////////////////////////////////////////////////////////////////
UpdateAuxTable(const GMM_DDI_UPDATEAUXTABLE * UpdateReq)474 GMM_STATUS GmmLib::GmmPageTableMgr::UpdateAuxTable(const GMM_DDI_UPDATEAUXTABLE *UpdateReq)
475 {
476 if(GetAuxL3TableAddr() == 0ULL)
477 {
478 GMM_ASSERTDPF(0, "Invalid AuxTable update request, AuxTable is not initialized");
479 return GMM_INVALIDPARAM;
480 }
481
482 if(!((UpdateReq->BaseResInfo->GetResFlags().Info.RenderCompressed ||
483 UpdateReq->BaseResInfo->GetResFlags().Info.MediaCompressed) &&
484 ((!UpdateReq->AuxResInfo && UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface) ||
485 (UpdateReq->AuxResInfo && UpdateReq->AuxResInfo->GetResFlags().Gpu.CCS))))
486 /*(UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource ||
487 UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth) */
488 //Allow Separate Aux for Depth/TR/MSAA/others?
489 {
490 GMM_ASSERTDPF(0, "Invalid AuxTable update request");
491 return GMM_INVALIDPARAM;
492 }
493
494 if(UpdateReq->Map && !(!UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource || (UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource && UpdateReq->UmdContext && UpdateReq->UmdContext->pCommandQueueHandle)))
495 {
496 //GMM_DPF_CRITICAL("TiledResources must Gpu-update AuxTable, proceeding with CPU-update...");
497
498 //Allowing CPU-update if requested so..
499 if(!UpdateReq->DoNotWait)
500 {
501 return GMM_INVALIDPARAM;
502 }
503 }
504
505 ENTER_CRITICAL_SECTION
506
507 if(UpdateReq->Map)
508 {
509 //Get AuxL1e data (other than CCS-adr) from main surface
510 uint64_t PartialL1e = AuxTTObj->CreateAuxL1Data(UpdateReq->BaseResInfo).Value;
511 GMM_STATUS Status = GMM_SUCCESS;
512
513 if(UpdateReq->BaseResInfo->GetResFlags().Gpu.TiledResource)
514 {
515 //Aux-TT is sparsely updated, for TRs, upon change in mapping state ie
516 // null->non-null must be mapped
517 // non-null->null invalidated on AuxTT
518 uint8_t CpuUpdate = UpdateReq->DoNotWait || !(UpdateReq->UmdContext && UpdateReq->UmdContext->pCommandQueueHandle);
519
520 GMM_GFX_ADDRESS AuxVA = UpdateReq->AuxSurfVA;
521 if(UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface)
522 {
523 GMM_UNIFIED_AUX_TYPE AuxType = GMM_AUX_CCS;
524 AuxType = (UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth && UpdateReq->BaseResInfo->GetResFlags().Gpu.CCS) ? GMM_AUX_ZCS : AuxType;
525 AuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset64(UpdateReq->BaseResInfo, AuxType);
526 }
527
528 }
529 else
530 {
531 GMM_GFX_ADDRESS AuxVA = {0};
532 GMM_GFX_ADDRESS UVAuxVA = {0};
533 GMM_GFX_SIZE_T YPlaneSize = 0;
534 uint32_t MaxPlanes = 1;
535
536 if(!UpdateReq->AuxResInfo && UpdateReq->BaseResInfo->GetResFlags().Gpu.UnifiedAuxSurface)
537 {
538 GMM_UNIFIED_AUX_TYPE AuxType = GMM_AUX_CCS;
539 AuxType = (UpdateReq->BaseResInfo->GetResFlags().Gpu.Depth &&
540 UpdateReq->BaseResInfo->GetResFlags().Gpu.CCS) ?
541 GMM_AUX_ZCS :
542 AuxType;
543
544 AuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset64(UpdateReq->BaseResInfo, AuxType);
545
546 //For UV Packed, Gen12 e2e compr supported formats have 2 planes per surface
547 //Each has distinct Aux surface, Y-plane/UV-plane must be mapped to respective Y/UV Aux surface
548 if(GmmIsPlanar(UpdateReq->BaseResInfo->GetResourceFormat()))
549 {
550 GMM_REQ_OFFSET_INFO ReqInfo = {0};
551 ReqInfo.Plane = GMM_PLANE_U;
552 ReqInfo.ReqRender = 1;
553
554 MaxPlanes = 2;
555 UpdateReq->BaseResInfo->GetOffset(ReqInfo);
556 YPlaneSize = ReqInfo.Render.Offset64;
557
558 UVAuxVA = UpdateReq->BaseGpuVA + GmmResGetAuxSurfaceOffset64(UpdateReq->BaseResInfo, GMM_AUX_UV_CCS);
559 }
560 }
561
562 //Per-plane Aux-TT map called with per-plane base/Aux address/size
563 for(uint32_t i = 0; i < MaxPlanes; i++)
564 {
565 GMM_GFX_SIZE_T SurfSize = (MaxPlanes > 1 && UpdateReq->BaseResInfo->GetArraySize() > 1) ?
566 (UpdateReq->BaseResInfo->GetQPitchPlanar(GMM_NO_PLANE) * UpdateReq->BaseResInfo->GetRenderPitch()) :
567 UpdateReq->BaseResInfo->GetSizeMainSurface();
568 GMM_GFX_SIZE_T MapSize = (i == 0) ? ((MaxPlanes > 1) ? YPlaneSize : SurfSize) : SurfSize - YPlaneSize;
569
570 GMM_GFX_ADDRESS BaseSurfVA = (UpdateReq->AuxResInfo || i == 0) ? UpdateReq->BaseGpuVA :
571 UpdateReq->BaseGpuVA + YPlaneSize;
572 GMM_GFX_ADDRESS AuxSurfVA = (UpdateReq->AuxResInfo) ? UpdateReq->AuxSurfVA : (i > 0 ? UVAuxVA : AuxVA);
573
574 //Luma plane reset LumaChroma bit
575 ((GMM_AUXTTL1e *)&PartialL1e)->LumaChroma = (i == 0) ? 0 : 1;
576 uint32_t ArrayEle = GFX_MAX(((MaxPlanes > 1) ?
577 UpdateReq->BaseResInfo->GetArraySize() :
578 1),
579 1);
580
581 for(uint32_t j = 0; j < ArrayEle; j++)
582 {
583 BaseSurfVA += ((j > 0) ? (UpdateReq->BaseResInfo->GetQPitchPlanar(GMM_PLANE_Y) * UpdateReq->BaseResInfo->GetRenderPitch()) : 0);
584 AuxSurfVA += (UpdateReq->AuxResInfo ?
585 ((j > 0) ? (UpdateReq->AuxResInfo->GetQPitchPlanar(GMM_PLANE_Y) * UpdateReq->BaseResInfo->GetRenderPitch()) : 0) :
586 ((j > 0) ? UpdateReq->BaseResInfo->GetAuxQPitch() : 0));
587
588 //(Flat mapping): Remove main/aux resInfo from params
589 Status = AuxTTObj->MapValidEntry(UpdateReq->UmdContext, BaseSurfVA, MapSize, UpdateReq->BaseResInfo,
590 AuxSurfVA, UpdateReq->AuxResInfo, PartialL1e, 1);
591 if(Status != GMM_SUCCESS)
592 {
593 GMM_ASSERTDPF(0, "Insufficient memory, free resources and try again");
594 EXIT_CRITICAL_SECTION
595 return Status;
596 }
597 }
598 }
599 }
600 }
601 else
602 {
603 //Invalidate all mappings for given main surface
604 AuxTTObj->InvalidateTable(UpdateReq->UmdContext, UpdateReq->BaseGpuVA, UpdateReq->BaseResInfo->GetSizeMainSurface(), UpdateReq->DoNotWait);
605 }
606
607 EXIT_CRITICAL_SECTION
608 return GMM_SUCCESS;
609 }
610
611 #if defined(__linux__) && !_WIN32
612 /////////////////////////////////////////////////////////////////////////////////////
613 /// Gets size of PageTable buffer object (BOs) list
614 ///
615 /// @param[in] TTFlags: Flags specifying PageTable-type for which BO-count required
616 /// @return non-zero if BO list is created. Zero otherwise.
617 /////////////////////////////////////////////////////////////////////////////////////
GetNumOfPageTableBOs(uint8_t TTFlags)618 int GmmLib::GmmPageTableMgr::GetNumOfPageTableBOs(uint8_t TTFlags)
619 {
620 int NumBO = 0;
621
622 __GMM_ASSERTPTR(TTFlags & AUXTT, 0);
623
624 ENTER_CRITICAL_SECTION
625
626 if(AuxTTObj && AuxTTObj->GetL3Handle())
627 NumBO++;
628
629 NumBO += NumNodePoolElements;
630
631 EXIT_CRITICAL_SECTION
632
633 return NumBO;
634 }
635
636 /////////////////////////////////////////////////////////////////////////////////////
637 /// Gets size of PageTable buffer object (BOs) list
638 ///
639 /// @param[in] TTFlags: Flags specifying PageTable-type for which BO-count required
640 /// @param[in][out] BOList: pointer to memory where PageTable BO*(s) must be sent
641 /// @return non-zero if BO list is created. Zero otherwise.
642 /////////////////////////////////////////////////////////////////////////////////////
GetPageTableBOList(uint8_t TTFlags,void * BOList)643 int GmmLib::GmmPageTableMgr::GetPageTableBOList(uint8_t TTFlags, void *BOList)
644 {
645 int NumBO = GetNumOfPageTableBOs(TTFlags);
646 HANDLE * Handles = (HANDLE *)BOList;
647 GmmLib::GMM_PAGETABLEPool *Pool;
648
649 __GMM_ASSERTPTR(TTFlags & AUXTT, 0);
650 __GMM_ASSERTPTR(BOList, 0);
651 __GMM_ASSERTPTR(NumBO, 0);
652
653 ENTER_CRITICAL_SECTION
654
655 if(AuxTTObj && AuxTTObj->GetL3Handle())
656 Handles[0] = AuxTTObj->GetL3Handle();
657
658 Pool = pPool;
659
660 for(int i = 0; i < NumNodePoolElements; i++)
661 {
662 if(Pool)
663 {
664 Handles[i + 1] = Pool->GetPoolHandle();
665 Pool = Pool->GetNextPool();
666 }
667 }
668
669 EXIT_CRITICAL_SECTION
670
671 return NumBO;
672 }
673 #endif
674
675 /////////////////////////////////////////////////////////////////////////////////////
676 /// Releases GmmPageTableMgr, deleting root-tables and existing page-table pools
677 /////////////////////////////////////////////////////////////////////////////////////
~GmmPageTableMgr()678 GmmLib::GmmPageTableMgr::~GmmPageTableMgr()
679 {
680 GMM_CLIENT ClientType;
681
682 GET_GMM_CLIENT_TYPE(pClientContext, ClientType);
683
684
685
686 if(pPool)
687 {
688 ENTER_CRITICAL_SECTION
689 pPool->__DestroyPageTablePool(&DeviceCbInt, hCsr);
690 NumNodePoolElements = 0;
691 EXIT_CRITICAL_SECTION
692 }
693
694 if(AuxTTObj)
695 {
696 DeleteCriticalSection(&PoolLock);
697
698 if(AuxTTObj)
699 {
700 if(AuxTTObj->NullL1Table)
701 {
702 delete AuxTTObj->NullL1Table;
703 }
704 if(AuxTTObj->NullL2Table)
705 {
706 delete AuxTTObj->NullL2Table;
707 }
708 AuxTTObj->DestroyL3Table();
709 delete AuxTTObj;
710 AuxTTObj = NULL;
711 }
712 }
713 }
714
715 /////////////////////////////////////////////////////////////////////////////////////
716 /// Instantiates and zeroes out GmmPageTableMgr
717 ///
718 /// @return GmmPageTableMgr*
719 /////////////////////////////////////////////////////////////////////////////////////
GmmPageTableMgr()720 GmmLib::GmmPageTableMgr::GmmPageTableMgr()
721 {
722 this->AuxTTObj = NULL;
723 this->pPool = NULL;
724 this->NumNodePoolElements = 0;
725 this->pClientContext = NULL;
726 this->hCsr = NULL;
727
728 memset(&DeviceCb, 0, sizeof(GMM_DEVICE_CALLBACKS));
729 memset(&DeviceCbInt, 0, sizeof(GMM_DEVICE_CALLBACKS_INT));
730 }
731
732
733