1 /*
2 * Copyright (c) 2015-2017, Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 //!
23 //! \file mhw_block_manager.cpp
24 //! \brief This modules implements memory block management functions as part of MHW dynamic state heap implementation
25 //!
26 #include "mhw_block_manager.h"
27 #include "mhw_utilities.h"
28 #include "mos_os_specific.h"
29
30 MHW_BLOCK_MANAGER_PARAMS MhwBlockManagerParams_default =
31 {
32 64, // Initial number of block objects in pool
33 1024, // Maximum number of block objects in pool
34 64, // Block pool increment size
35 0x00080000, // Initial heap size (512k)
36 0x00080000, // Heap size increment (512k)
37 0x01000000, // Maximum overall heap size (16M)
38 32, // Maximum number of heaps (32) (512k x 32 = 16M)
39 0x0800, // Block granularity = 2k (also represents min block alignment)
40 0x0800 // Min free block size = 2k (to reduce fragmentation)
41 };
42
43 const char *szListName[MHW_BLOCK_STATE_COUNT] = {
44 "POOL",
45 "FREE",
46 "ALLOCATED",
47 "SUBMITTED",
48 "DELETED"
49 };
50
Mhw_BlockManager_ReverseMergeSort_With_Index(const uint32_t * pdwSizes,int32_t iCount,uint8_t * pSortedIndex)51 void Mhw_BlockManager_ReverseMergeSort_With_Index(const uint32_t *pdwSizes, int32_t iCount, uint8_t *pSortedIndex)
52 {
53 uint8_t i, n;
54 uint8_t *pSrc, *pDst; // Source and Destination groups (alternate)
55 uint8_t Index1[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted indexes 1
56 uint8_t Index2[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted indexes 2 (alternating with Index1)
57 uint8_t *s1, *s2; // Merge source groups 1 and 2
58 uint8_t n1, n2; // Merge sizes groups 1 and 2
59 uint8_t *d; // Merge destination
60
61 // Very simple cases
62 if (iCount <= 1)
63 {
64 pSortedIndex[0] = 0;
65 return;
66 }
67 else if (iCount == 2)
68 {
69 pSortedIndex[0] = (pdwSizes[0] < pdwSizes[1]) ? 1 : 0;
70 pSortedIndex[1] = 1 - pSortedIndex[0];
71 return;
72 }
73
74 // Initialize sorted index table
75 for (i = 0; i < iCount; i++)
76 {
77 Index1[i] = i;
78 }
79
80 // Start alternating buffers (last will be the actual output)
81 pSrc = Index1;
82 pDst = Index2;
83
84 // Merge sort algorithm:
85 // Sort will perform O(log n) passes; first pass sorts (iCount/2) groups of 2, then (iCount/4) groups of 4, and so on.
86 // Each pass requires traversal of the entire list - O(n)
87 // Algorithm is expected to be O(n * log n)
88 for (n = 1; n < iCount; n *= 2)
89 {
90 // Setup sorted target output
91 if (n*2 < iCount)
92 {
93 d = pDst;
94 }
95 else
96 { // Last pass, output goes to caller
97 d = pSortedIndex;
98 }
99
100 // Group selection and merge - O(n)
101 s1 = pSrc - n; // First group
102 s2 = pSrc; // Second group
103 for (i = n; i < iCount; i += 2*n) // i is the offset of the 2nd group
104 {
105 s1 += n;
106 n1 = n;
107 s2 += n;
108 n2 = n;
109
110 // Limit size of last group
111 if (i + n > iCount)
112 {
113 n2 = iCount - i;
114 }
115
116 // Merge groups
117 while (n1 > 0 && n2 > 0)
118 {
119 if (pdwSizes[*s1] >= pdwSizes[*s2])
120 {
121 *(d++) = *(s1++);
122 n1--;
123 }
124 else
125 {
126 *(d++) = *(s2++);
127 n2--;
128 }
129 }
130
131 // Merge remaining items
132 while (n1 > 0)
133 {
134 *(d++) = *s1++;
135 n1--;
136 }
137 while (n2 > 0)
138 {
139 *(d++) = *s2++;
140 n2--;
141 }
142 }
143
144 // Copy the last group (not merge-sorted)
145 for (i = i - n; i < iCount; i++)
146 {
147 *(d++) = *(s2++);
148 }
149
150 // New pass, switch Src/Dst sorted index buffers
151 d = pDst;
152 pDst = pSrc;
153 pSrc = d;
154 }
155 }
156
Mhw_BlockManager_ReverseMergeSort(uint32_t * pdwSizes,int32_t iCount)157 void Mhw_BlockManager_ReverseMergeSort(uint32_t *pdwSizes, int32_t iCount)
158 {
159 uint8_t i, n;
160 uint32_t *pSrc, *pDst; // Source and Destination groups (alternate)
161 uint32_t Buffer1[2 * MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted buffer 1
162 uint32_t Buffer2[2 * MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // Temporary sorted buffer 1
163 uint32_t *s1, *s2; // Merge source groups 1 and 2
164 uint8_t n1, n2; // Merge sizes groups 1 and 2
165 uint32_t *d; // Merge destination
166
167 // Very simple cases
168 if (iCount <= 1)
169 {
170 return;
171 }
172 else if (iCount == 2)
173 {
174 if (pdwSizes[0] < pdwSizes[1])
175 {
176 uint32_t tmp = pdwSizes[1];
177 pdwSizes[1] = pdwSizes[0];
178 pdwSizes[0] = tmp;
179 }
180 return;
181 }
182 else if (iCount > 2 * MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY)
183 {
184 return;
185 }
186
187 // Merge sort algorithm:
188 // Sort will perform O(log n) passes; first pass sorts (iCount/2) groups of 2, then (iCount/4) groups of 4, and so on.
189 // Each pass requires traversal of the entire list - O(n)
190 // Algorithm is expected to be O(n * log n)
191 pSrc = Buffer1;
192 pDst = Buffer2;
193 for (n = 1; n < iCount; n *= 2)
194 {
195 if (n == 1)
196 {
197 s1 = pdwSizes - 1;
198 s2 = pdwSizes;
199 }
200 else
201 {
202 s1 = pSrc - n;
203 s2 = pSrc;
204 }
205
206 // Setup sorted target output
207 if (n*2 < iCount)
208 {
209 d = pDst;
210 }
211 else
212 { // Last pass, output goes to caller
213 d = pdwSizes;
214 }
215
216 // Group selection and merge - O(n)
217 for (i = n; i < iCount; i += 2*n) // i is the offset of the 2nd group
218 {
219 s1 += n;
220 n1 = n;
221 s2 += n;
222 n2 = n;
223
224 // Limit size of last group
225 if (i + n > iCount)
226 {
227 n2 = iCount - i;
228 }
229
230 // Merge groups
231 while (n1 > 0 && n2 > 0)
232 {
233 if (*s1 >= *s2)
234 {
235 *(d++) = *(s1++);
236 n1--;
237 }
238 else
239 {
240 *(d++) = *(s2++);
241 n2--;
242 }
243 }
244
245 // Merge remaining items
246 while (n1 > 0)
247 {
248 *(d++) = *s1++;
249 n1--;
250 }
251 while (n2 > 0)
252 {
253 *(d++) = *s2++;
254 n2--;
255 }
256 }
257
258 // Copy the last group (not merge-sorted)
259 for (i = i - n; i < iCount; i++)
260 {
261 *(d++) = *(s2++);
262 }
263
264 // New pass, switch Src/Dst
265 d = pDst;
266 pDst = pSrc;
267 pSrc = d;
268 }
269 }
270
MHW_BLOCK_MANAGER(PMHW_BLOCK_MANAGER_PARAMS pParams)271 MHW_BLOCK_MANAGER::MHW_BLOCK_MANAGER(PMHW_BLOCK_MANAGER_PARAMS pParams):
272 m_MemoryPool(sizeof(MHW_STATE_HEAP_MEMORY_BLOCK), sizeof(void *)),
273 m_pStateHeap(nullptr)
274 {
275 //Init Parameters
276 if(pParams != nullptr)
277 {
278 m_Params = *pParams;
279 }
280 else
281 {
282 m_Params = MhwBlockManagerParams_default;
283 }
284
285 //Init Memory block list
286 for (int32_t i = (int32_t)MHW_BLOCK_STATE_POOL; i < MHW_BLOCK_STATE_COUNT; i++)
287 {
288 MOS_ZeroMemory(&m_BlockList[i], sizeof(MHW_BLOCK_LIST));
289 m_BlockList[i].BlockState = (MHW_BLOCK_STATE) i;
290 m_BlockList[i].pBlockManager = this;
291 MOS_SecureStrcpy(m_BlockList[i].szListName, 16, szListName[i]);
292 }
293
294 //Extend Pool
295 ExtendPool(m_Params.dwPoolInitialCount);
296 }
297
~MHW_BLOCK_MANAGER()298 MHW_BLOCK_MANAGER::~MHW_BLOCK_MANAGER()
299 {
300
301 }
302
SetStateHeap(PMHW_STATE_HEAP pStateHeap)303 void MHW_BLOCK_MANAGER::SetStateHeap(PMHW_STATE_HEAP pStateHeap)
304 {
305 if (pStateHeap)
306 {
307 m_pStateHeap = pStateHeap;
308 }
309 return;
310 }
311
RegisterStateHeap(PMHW_STATE_HEAP pStateHeap)312 MOS_STATUS MHW_BLOCK_MANAGER::RegisterStateHeap(
313 PMHW_STATE_HEAP pStateHeap)
314 {
315 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
316
317 // Setup state heap associated with the memory block manager
318 if (!m_pStateHeap)
319 {
320 m_pStateHeap = pStateHeap;
321 }
322
323 // Indicates that state heap is now being managed by this block manager object
324 pStateHeap->pBlockManager = this;
325
326 // Get memory block object to represent the state heap memory (free)
327 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = GetBlockFromPool();
328 if (!pBlock)
329 {
330 return MOS_STATUS_NO_SPACE;
331 }
332
333 // Setup block parameters
334 pBlock->pStateHeap = pStateHeap; // Points to state heap
335 pBlock->pHeapPrev = nullptr; // First Block in current heap
336 pBlock->pHeapNext = nullptr; // Last Block in current heap
337 pBlock->dwOffsetInStateHeap = 0; // Offset is 0 = base of state heap
338 pBlock->dwBlockSize = pStateHeap->dwSize; // Size of the entire state heap
339 FrameTrackerTokenFlat_Validate(&pBlock->trackerToken);
340 pBlock->bStatic = 0;
341
342 // Set first/last memory block in state heap
343 pStateHeap->pMemoryHead = pBlock;
344 pStateHeap->pMemoryTail = pBlock;
345
346 // Reset special state heap controls
347 pStateHeap->pDebugKernel = nullptr; // Debug kernel loaded in this heap (if ISH)
348 pStateHeap->pScratchSpace = nullptr; // Active scratch space in this heap (if GSH)
349 pStateHeap->dwScratchSpace = 0; // Current scratch space size in this heap
350
351 // Insert block at the end of the free memory pool
352 AttachBlock(MHW_BLOCK_STATE_FREE, pBlock, MHW_BLOCK_POSITION_TAIL);
353
354 return eStatus;
355 }
356
UnregisterStateHeap(PMHW_STATE_HEAP pStateHeap)357 MOS_STATUS MHW_BLOCK_MANAGER::UnregisterStateHeap(
358 PMHW_STATE_HEAP pStateHeap)
359 {
360 MHW_STATE_HEAP_MEMORY_BLOCK *pBlock;
361
362 bool bReleaseHeap = true;
363
364 // Verification loop - check if heap can be freed
365 for (pBlock = pStateHeap->pMemoryHead; pBlock != nullptr; pBlock = pBlock->pHeapNext)
366 {
367 // Move blocks not in use to deleted queue - so they cannot be reused
368 // NOTE: Blocks in SUBMITTED state need to wait for completion before releasing
369 if (pBlock->BlockState == MHW_BLOCK_STATE_FREE ||
370 pBlock->BlockState == MHW_BLOCK_STATE_ALLOCATED)
371 {
372 // Update heap usage
373 if (pBlock->BlockState == MHW_BLOCK_STATE_FREE)
374 {
375 pStateHeap->dwFree -= pBlock->dwBlockSize;
376 }
377 else
378 {
379 pStateHeap->dwUsed -= pBlock->dwBlockSize;
380 }
381
382 DetachBlock(pBlock->BlockState, pBlock);
383 AttachBlock(MHW_BLOCK_STATE_DELETED, pBlock, MHW_BLOCK_POSITION_TAIL);
384 }
385 // Block is allocated or submitted - mark block for deletion when released
386 else if (pBlock->BlockState != MHW_BLOCK_STATE_DELETED)
387 {
388 pBlock->bStatic = false; // Unlock block
389 pBlock->bDelete = true; // Mark block for deletion when done
390 bReleaseHeap = false; // Heap cannot be released at this moment
391 }
392 }
393
394 // All blocks have been sucessfully deleted -> move all blocks to pool, unregister state heap
395 if (bReleaseHeap)
396 {
397 // Return deleted blocks back to pool
398 for (pBlock = pStateHeap->pMemoryHead; pBlock != nullptr; pBlock = pBlock->pHeapNext)
399 {
400 // Sanity check - all blocks must be in this state!
401 if (pBlock->BlockState == MHW_BLOCK_STATE_DELETED)
402 {
403 DetachBlock(MHW_BLOCK_STATE_DELETED, pBlock);
404 ReturnBlockToPool(pBlock);
405 }
406 else
407 {
408 // NEVER SUPPOSED TO HAPPEN DUE TO PREVIOUS LOOP
409 MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_UnregisterStateHeap: Invalid state, heap blocks are supposed to be all deleted by now");
410 }
411 }
412 return MOS_STATUS_SUCCESS;
413 }
414 else
415 { // State heap cannot be unregistered because some blocks are still in use
416 return MOS_STATUS_UNKNOWN;
417 }
418 }
419
GetBlockFromPool()420 PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::GetBlockFromPool()
421 {
422 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr;
423
424 // Ran out of memory blocks... extend pool of memory block objects
425 if (m_BlockList[MHW_BLOCK_STATE_POOL].iCount == 0)
426 {
427 ExtendPool(m_Params.dwPoolIncrement);
428 }
429
430 // Retrieve block object from head of the pool
431 pBlock = DetachBlock(MHW_BLOCK_STATE_POOL, MHW_BLOCK_POSITION_HEAD);
432
433 return pBlock;
434 }
435
ExtendPool(uint32_t dwCount)436 void MHW_BLOCK_MANAGER::ExtendPool(uint32_t dwCount)
437 {
438 uint32_t dwBlockID = m_MemoryPool.m_dwObjCount; // Block ID starts from the current block count
439
440 // Limits the number of memory blocks
441 if (m_MemoryPool.m_dwCount + dwCount > m_Params.dwPoolMaxCount)
442 {
443 dwCount = m_Params.dwPoolMaxCount - m_MemoryPool.m_dwCount;
444 }
445
446 // Extend pool of block objects
447 if (dwCount > 0)
448 {
449 // Allocate array of block objects into the pool
450 MHW_STATE_HEAP_MEMORY_BLOCK *pBlockArray = (MHW_STATE_HEAP_MEMORY_BLOCK *) (m_MemoryPool.Allocate(dwCount));
451 if (pBlockArray)
452 {
453 // Insert newly created block objects into the linked list of pool objects available to the memory block manager
454 for (; dwCount > 0; dwCount--, pBlockArray++)
455 {
456 pBlockArray->dwBlockSize = 0;
457 pBlockArray->pPrev = pBlockArray->pNext = nullptr;
458 pBlockArray->Reserved = dwBlockID++;
459 AttachBlock(MHW_BLOCK_STATE_POOL, pBlockArray, MHW_BLOCK_POSITION_TAIL);
460 }
461 }
462 }
463 }
464
AttachBlock(MHW_BLOCK_STATE BlockState,PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos)465 MOS_STATUS MHW_BLOCK_MANAGER::AttachBlock(
466 MHW_BLOCK_STATE BlockState,
467 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,
468 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos)
469 {
470 PMHW_BLOCK_LIST pList;
471 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
472
473 // Verify parameters - objects cannot be null, block state must be valid
474 if (pBlock == nullptr ||
475 BlockState < MHW_BLOCK_STATE_POOL ||
476 BlockState >= MHW_BLOCK_STATE_COUNT)
477 {
478 return MOS_STATUS_INVALID_PARAMETER;
479 }
480
481 // Fails if block is still attached to a list
482 if (pBlock->pPrev || pBlock->pNext)
483 {
484 return MOS_STATUS_INVALID_PARAMETER;
485 }
486
487 // Get list associated with block state; move block to the list
488 pList = &m_BlockList[BlockState];
489 BLOCK_MANAGER_CHK_STATUS(AttachBlockInternal(pList, BlockState, pBlock, pBlockPos));
490
491 return MOS_STATUS_SUCCESS;
492 }
493
AttachBlockInternal(PMHW_BLOCK_LIST pList,MHW_BLOCK_STATE BlockState,PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos)494 MOS_STATUS MHW_BLOCK_MANAGER::AttachBlockInternal(
495 PMHW_BLOCK_LIST pList,
496 MHW_BLOCK_STATE BlockState,
497 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,
498 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos)
499 {
500
501 // Check if this is the correct list!
502 if (pList->BlockState != BlockState)
503 {
504 MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_AttachBlock_Internal: Block state doesn't match the list provided");
505 return MOS_STATUS_INVALID_PARAMETER;
506 }
507
508 // Setup the block state
509 pBlock->BlockState = BlockState;
510
511 // Attaches to the head of the list
512 if (pBlockPos == MHW_BLOCK_POSITION_TAIL)
513 {
514 pBlock->pPrev = pList->pTail;
515 pBlock->pNext = nullptr;
516 }
517 // Attaches to the tail of the list
518 else if (pBlockPos == MHW_BLOCK_POSITION_HEAD)
519 {
520 pBlock->pPrev = nullptr;
521 pBlock->pNext = pList->pHead;
522 }
523 // Insert after block provided - ensures that it belongs to the same list
524 else if (pBlockPos->BlockState == BlockState)
525 {
526 pBlock->pPrev = pBlockPos;
527 pBlock->pNext = pBlockPos->pNext;
528 }
529 // Insertion point does not belong to the same list
530 else
531 {
532 MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_AttachBlock_Internal: Reference block does not belong to the list provided");
533 return MOS_STATUS_INVALID_PARAMETER;
534 }
535
536 // Modify previous block or head of the list
537 if (pBlock->pPrev)
538 {
539 pBlock->pPrev->pNext = pBlock;
540 }
541 else
542 {
543 pList->pHead = pBlock;
544 }
545
546 // Modify next block or tail of the list
547 if (pBlock->pNext)
548 {
549 pBlock->pNext->pPrev = pBlock;
550 }
551 else
552 {
553 pList->pTail = pBlock;
554 }
555
556 // Track size and number of blocks in the list
557 pList->dwSize += pBlock->dwBlockSize;
558 pList->iCount++;
559
560 return MOS_STATUS_SUCCESS;
561 }
562
DetachBlock(MHW_BLOCK_STATE BlockState,PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos)563 PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::DetachBlock(
564 MHW_BLOCK_STATE BlockState,
565 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos)
566 {
567 PMHW_BLOCK_LIST pList;
568 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr;
569
570 // Verify parameters - object cannot be null, block state must be valid
571 if (BlockState < MHW_BLOCK_STATE_POOL ||
572 BlockState >= MHW_BLOCK_STATE_COUNT)
573 {
574 return nullptr;
575 }
576
577 // Get list associated with block state
578 pList = &m_BlockList[BlockState];
579
580 // Remove block from list, performing sanity check (check if block is in correct list)
581 pBlock = DetachBlockInternal(pList, pBlockPos);
582
583 return pBlock;
584 }
585
DetachBlockInternal(PMHW_BLOCK_LIST pList,PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)586 PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::DetachBlockInternal(
587 PMHW_BLOCK_LIST pList,
588 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)
589 {
590
591 if(pList == nullptr)
592 {
593 return nullptr;
594 }
595
596 #ifdef MHW_DYNAMIC_STATE_HEAP_LOGGING
597 const char *szPos = "REF ";
598 if (pBlock == MHW_BLOCK_POSITION_HEAD)
599 szPos = "HEAD";
600 else if (pBlock == MHW_BLOCK_POSITION_TAIL)
601 szPos = "TAIL";
602 #endif
603
604 // Get block from the head of the list
605 if (pBlock == MHW_BLOCK_POSITION_HEAD)
606 {
607 pBlock = pList->pHead;
608 }
609 // Get block from the tail of the list
610 else if (pBlock == MHW_BLOCK_POSITION_TAIL)
611 {
612 pBlock = pList->pTail;
613 }
614 // Block does not belong to correct list - ASSERT and get block from head of the list
615 else if (pBlock->BlockState != pList->BlockState)
616 {
617 MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_DetachBlock_Internal: Block provided does not belong to the list provided");
618 pBlock = nullptr;
619 }
620
621 if (!pBlock)
622 {
623 return nullptr;
624 }
625
626 // Detach block from previous; if first, update head of the list
627 if (pBlock->pPrev)
628 {
629 pBlock->pPrev->pNext = pBlock->pNext;
630 }
631 else
632 {
633 pList->pHead = pBlock->pNext;
634 }
635
636 // Detach block from next; if last, update tail of the list
637 if (pBlock->pNext)
638 {
639 pBlock->pNext->pPrev = pBlock->pPrev;
640 }
641 else
642 {
643 pList->pTail = pBlock->pPrev;
644 }
645
646 // reset pointers - block is detached
647 pBlock->pNext = pBlock->pPrev = nullptr;
648
649 // track size and number of block in the list
650 pList->dwSize -= pBlock->dwBlockSize;
651 pList->iCount--;
652
653 return pBlock;
654 }
655
MoveBlock(PMHW_BLOCK_LIST pSrcList,PMHW_BLOCK_LIST pDstList,PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos)656 MOS_STATUS MHW_BLOCK_MANAGER::MoveBlock(
657 PMHW_BLOCK_LIST pSrcList, // Source list
658 PMHW_BLOCK_LIST pDstList, // Destination list
659 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, // Block to be moved (or HEAD/TAIL of source list)
660 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockPos) // Position to insert (or HEAD/TAIL of target list)
661 {
662 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
663
664 BLOCK_MANAGER_CHK_NULL(pSrcList);
665 BLOCK_MANAGER_CHK_NULL(pDstList);
666
667 pBlock = DetachBlockInternal(pSrcList, pBlock);
668 BLOCK_MANAGER_CHK_NULL(pBlock);
669
670 eStatus = AttachBlockInternal(pDstList, pDstList->BlockState, pBlock, pBlockPos);
671
672 if (eStatus != MOS_STATUS_SUCCESS)
673 {
674 MHW_ASSERTMESSAGE("ERROR: Mhw_BlockManager_MoveBlock_Internal: Failed to move block");
675 AttachBlockInternal(pDstList, pDstList->BlockState, pBlock, MHW_BLOCK_POSITION_TAIL);
676 }
677
678 return eStatus;
679 }
680
ReturnBlockToPool(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)681 void MHW_BLOCK_MANAGER::ReturnBlockToPool(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)
682 {
683 pBlock->dwBlockSize = 0;
684 pBlock->pPrev = pBlock->pNext = nullptr;
685
686 AttachBlock(MHW_BLOCK_STATE_POOL, pBlock, MHW_BLOCK_POSITION_TAIL);
687
688 return;
689 }
690
Refresh()691 MOS_STATUS MHW_BLOCK_MANAGER::Refresh()
692 {
693 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock, pNext;
694 PMHW_BLOCK_LIST pList;
695 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
696
697 // Refresh status of SUBMITTED blocks
698 pList = &m_BlockList[MHW_BLOCK_STATE_SUBMITTED];
699 pNext = nullptr;
700 for (pBlock = pList->pHead; pBlock != nullptr; pBlock = pNext)
701 {
702 // check of block may be released - if not, continue loop
703 // NOTE - blocks are to be inserted in the sequence of execution,
704 // so we may finish the search as soon as we find the first
705 // block still "in use" (block tag > current sync tag)
706 // For now we're doing an exhaustive search, but it may not be needed - once
707 // we find the first block still in execution, we can probably stop the search
708
709 // Save next pointer before moving the block to another queue
710 pNext = pBlock->pNext;
711
712 // Check if block is still in use, if so, continue search
713 // NOTE - the following expression avoids sync tag wrapping around MAX_INT -> 0
714 if (!FrameTrackerTokenFlat_IsExpired(&pBlock->trackerToken))
715 {
716 continue;
717 }
718
719 // Block is flagged for deletion
720 if (pBlock->bDelete)
721 {
722 BLOCK_MANAGER_CHK_STATUS(FreeBlock(pBlock));
723 }
724 // Block is static -> move back to allocated list
725 else if (pBlock->bStatic)
726 {
727 BLOCK_MANAGER_CHK_STATUS(MoveBlock(pList, &m_BlockList[MHW_BLOCK_STATE_ALLOCATED],
728 pBlock, MHW_BLOCK_POSITION_TAIL));
729 }
730 else
731 // Block is non-static -> free block
732 {
733 FreeBlock(pBlock);
734 }
735 }
736
737 return eStatus;
738 }
739
ConsolidateBlock(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)740 void MHW_BLOCK_MANAGER::ConsolidateBlock(
741 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)
742 {
743 PMHW_STATE_HEAP_MEMORY_BLOCK pAux;
744
745 // Check input parameters
746 if (!pBlock || pBlock->BlockState != MHW_BLOCK_STATE_FREE)
747 {
748 return;
749 }
750
751 // Consolidate pBlock with previous blocks
752 PMHW_BLOCK_LIST pFree = &m_BlockList[MHW_BLOCK_STATE_FREE];
753 for (pAux = pBlock->pHeapPrev; (pAux != nullptr) && (pAux->BlockState == MHW_BLOCK_STATE_FREE); pAux = pBlock->pHeapPrev)
754 {
755 // Remove block from free block list
756 DetachBlock(MHW_BLOCK_STATE_FREE, pAux);
757
758 // Absorb free space into original block by adjusting offset/size
759 pBlock->dwOffsetInStateHeap -= pAux->dwBlockSize;
760 pBlock->dwBlockSize += pAux->dwBlockSize;
761 pFree->dwSize += pAux->dwBlockSize;
762
763 // Detach memory block from sequential memory list
764 pBlock->pHeapPrev = pAux->pHeapPrev;
765 if (pBlock->pHeapPrev)
766 {
767 pBlock->pHeapPrev->pHeapNext = pBlock;
768 }
769 else
770 {
771 pBlock->pStateHeap->pMemoryHead = pBlock;
772 }
773
774 // Memory block object no longer needed - return to pool after consolidation
775 ReturnBlockToPool(pAux);
776 }
777
778 // Consolidate pBlock with following blocks
779 for (pAux = pBlock->pHeapNext; (pAux != nullptr) && (pAux->BlockState == MHW_BLOCK_STATE_FREE); pAux = pBlock->pHeapNext)
780 {
781 // Remove block from free block list
782 DetachBlock(MHW_BLOCK_STATE_FREE, pAux);
783
784 // Absorb free space into original block by adjusting block size (offset remains the same)
785 pBlock->dwBlockSize += pAux->dwBlockSize;
786 pFree->dwSize += pAux->dwBlockSize;
787
788 // Detach memory block from sequential memory list
789 pBlock->pHeapNext = pAux->pHeapNext;
790 if (pBlock->pHeapNext)
791 {
792 pBlock->pHeapNext->pHeapPrev = pBlock;
793 }
794 else
795 {
796 pBlock->pStateHeap->pMemoryTail = pBlock;
797 }
798
799 // Memory block object no longer needed - return to pool after consolidation
800 ReturnBlockToPool(pAux);
801 }
802 }
803
AllocateBlockInternal(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,uint32_t dwAlignment)804 MOS_STATUS MHW_BLOCK_MANAGER::AllocateBlockInternal(
805 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,
806 uint32_t dwAlignment)
807 {
808 PMHW_STATE_HEAP pHeap;
809 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
810 BLOCK_MANAGER_CHK_NULL(pBlock);
811
812 // Remove block from free list - if block is not in free list, this operation will fail and return nullptr
813 pBlock = DetachBlock( MHW_BLOCK_STATE_FREE, pBlock);
814 BLOCK_MANAGER_CHK_NULL(pBlock);
815
816 // Initialize block data structures
817 pBlock->bDelete = false;
818
819 pBlock->dwDataOffset = MOS_ALIGN_CEIL(pBlock->dwOffsetInStateHeap, dwAlignment);
820 pBlock->dwAlignment = pBlock->dwDataOffset - pBlock->dwOffsetInStateHeap;
821 pBlock->dwDataSize = pBlock->dwBlockSize - pBlock->dwAlignment;
822 pBlock->pDataPtr = (uint8_t*)pBlock->pStateHeap->pvLockedHeap + pBlock->dwDataOffset;
823
824 // Move block to allocated list
825 AttachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlock, MHW_BLOCK_POSITION_TAIL);
826
827 // Update available space in heap
828 pHeap = pBlock->pStateHeap;
829 pHeap->dwFree -= pBlock->dwBlockSize;
830 pHeap->dwUsed += pBlock->dwBlockSize;
831
832 return MOS_STATUS_SUCCESS;
833 }
834
SplitBlockInternal(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,uint32_t dwSplitSize,uint32_t dwAlignment,bool bBackward)835 MOS_STATUS MHW_BLOCK_MANAGER::SplitBlockInternal(
836 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,
837 uint32_t dwSplitSize,
838 uint32_t dwAlignment,
839 bool bBackward)
840 {
841 uint32_t dwSplitOffset = 0;
842 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
843 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockL, pBlockH;
844
845 BLOCK_MANAGER_CHK_NULL(pBlock);
846
847 // Split cannot be less than min block size allowed
848 dwSplitSize = MOS_MAX(dwSplitSize, m_Params.dwHeapBlockMinSize);
849 if (pBlock->dwBlockSize < dwSplitSize)
850 {
851 return MOS_STATUS_UNKNOWN;
852 }
853
854 // Verify block state
855 if (pBlock->BlockState <= MHW_BLOCK_STATE_POOL || // Cannot split a block object from pool (contains invalid data)
856 pBlock->BlockState >= MHW_BLOCK_STATE_DELETED) // Cannot split a block being deleted
857 {
858 return MOS_STATUS_INVALID_PARAMETER;
859
860 }
861
862 // Select list
863 PMHW_BLOCK_LIST pList = &(m_BlockList[pBlock->BlockState]);
864
865 if (bBackward)
866 {
867 // Split FROM THE END of the block (higher offset)
868 dwSplitOffset = MOS_ALIGN_FLOOR(pBlock->dwOffsetInStateHeap + pBlock->dwBlockSize - dwSplitSize, dwAlignment);
869 dwSplitOffset = MOS_ALIGN_FLOOR(dwSplitOffset, m_Params.dwHeapGranularity);
870 }
871 else
872 {
873 // Split FROM THE BEGINNING of the block (lower offset)
874 dwSplitOffset = MOS_ALIGN_CEIL(pBlock->dwOffsetInStateHeap, dwAlignment);
875 dwSplitOffset = MOS_ALIGN_CEIL(dwSplitOffset + dwSplitSize, m_Params.dwHeapGranularity);
876 }
877
878 // Fail if block cannot be split
879 if (dwSplitOffset < pBlock->dwOffsetInStateHeap + m_Params.dwHeapBlockMinSize || // First fragment is too small
880 pBlock->dwOffsetInStateHeap + pBlock->dwBlockSize < dwSplitOffset + m_Params.dwHeapBlockMinSize) // Second fragment is too small
881 {
882 return MOS_STATUS_UNKNOWN;
883 }
884
885 if (bBackward)
886 {
887 pBlockH = pBlock; // We'll keep the high end of the block
888 pBlockL = GetBlockFromPool();
889 BLOCK_MANAGER_CHK_NULL(pBlockL);
890
891 uint32_t reserved = pBlockL->Reserved;
892 *pBlockL = *pBlock;
893 pBlockL->Reserved = reserved;
894
895 if (pBlock->pPrev)
896 {
897 pBlock->pPrev->pNext = pBlockL;
898 }
899 else
900 {
901 pList->pHead = pBlockL;
902 }
903
904 if (pBlock->pHeapPrev)
905 {
906 pBlock->pHeapPrev->pHeapNext = pBlockL;
907 }
908 else
909 {
910 pBlock->pStateHeap->pMemoryHead = pBlockL;
911 }
912 }
913 else
914 {
915 pBlockL = pBlock; // We'll keep the low end of the block
916 pBlockH = GetBlockFromPool();
917 BLOCK_MANAGER_CHK_NULL(pBlockH);
918
919 uint32_t reserved = pBlockH->Reserved;
920 *pBlockH = *pBlock;
921 pBlockH->Reserved = reserved;
922
923 if (pBlock->pNext)
924 {
925 pBlock->pNext->pPrev = pBlockH;
926 }
927 else
928 {
929 pList->pTail = pBlockH;
930 }
931
932 if (pBlock->pHeapNext)
933 {
934 pBlock->pHeapNext->pHeapPrev = pBlockH;
935 }
936 else
937 {
938 pBlock->pStateHeap->pMemoryTail = pBlockH;
939 }
940 }
941
942 // Update block adjacency list
943 pBlockL->pHeapNext = pBlockH;
944 pBlockH->pHeapPrev = pBlockL;
945
946 // Ensures that the new block is tracked in the same list as the parent block, update block count
947 pList->iCount++;
948 pBlockL->pNext = pBlockH;
949 pBlockH->pPrev = pBlockL;
950
951 // Adjust Block sizes
952 pBlockL->dwBlockSize = dwSplitOffset - pBlockL->dwOffsetInStateHeap; // Updates L block size based on split offset
953 pBlockH->dwOffsetInStateHeap = dwSplitOffset; // Sets 2nd block offset
954 pBlockH->dwBlockSize -= pBlockL->dwBlockSize; // Updates H block size by subtracting L block size
955
956 // Adjust Block data related pointers/sizes only if block is not free
957 if (pBlockL->BlockState != MHW_BLOCK_STATE_FREE)
958 {
959 pBlockL->dwDataSize -= pBlockH->dwBlockSize; // Removes size of new block from amount of data available
960 pBlockH->dwDataOffset = MOS_ALIGN_CEIL(dwSplitOffset, dwAlignment); // Adjust offset to data (accounting for alignment)
961 pBlockH->dwAlignment = pBlockH->dwDataOffset - dwSplitOffset; // Calculate alignment shift
962 pBlockH->dwDataSize = pBlockH->dwBlockSize - dwAlignment; // Adjust amount of data available
963 pBlockH->pDataPtr = (uint8_t*)pBlockH->pStateHeap->pvLockedHeap + pBlockH->dwDataOffset; // Setup pointer to data (the heap is locked)
964 }
965
966 return eStatus;
967 }
968
MergeBlocksInternal(PMHW_STATE_HEAP_MEMORY_BLOCK pBlockL,PMHW_STATE_HEAP_MEMORY_BLOCK pBlockH,uint32_t dwAlignment,bool bBackward)969 MOS_STATUS MHW_BLOCK_MANAGER::MergeBlocksInternal(
970 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockL, // block in lower memory
971 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockH, // block in higher memory
972 uint32_t dwAlignment, // final block alignment
973 bool bBackward) // true if pBlockL (free) is merged into pBlockH; false if pBlockH (free) is merged into pBlockL
974 {
975 PMHW_BLOCK_LIST pList;
976 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
977
978 BLOCK_MANAGER_CHK_NULL(pBlockL);
979 BLOCK_MANAGER_CHK_NULL(pBlockH);
980
981 // Blocks must be contiguous in memory
982 if (pBlockL->pHeapNext != pBlockH ||
983 pBlockH->pHeapPrev != pBlockL)
984 {
985 return MOS_STATUS_INVALID_PARAMETER;
986 }
987
988 // 1st block (L) is merged into 2nd (H)
989 if (bBackward)
990 {
991 if (pBlockL->BlockState != MHW_BLOCK_STATE_FREE || // 1st block must be free
992 pBlockH->BlockState < MHW_BLOCK_STATE_FREE || // 2nd block cannot be in pool
993 pBlockH->BlockState > MHW_BLOCK_STATE_SUBMITTED) // or deleted
994 {
995 return MOS_STATUS_INVALID_PARAMETER;
996 }
997
998 // Merge blocks
999 pBlockL = DetachBlock(MHW_BLOCK_STATE_FREE, pBlockL);
1000 BLOCK_MANAGER_CHK_NULL(pBlockL);
1001
1002 pBlockH->dwOffsetInStateHeap = pBlockL->dwOffsetInStateHeap;
1003 pBlockH->dwBlockSize += pBlockL->dwBlockSize;
1004
1005 // Add size to the target block list
1006 pList = &(m_BlockList[pBlockH->BlockState]);
1007 pList->dwSize += pBlockL->dwBlockSize;
1008
1009 // If block allocated or submitted, adjust data references (don't care if block is free)
1010 if (pBlockH->BlockState != MHW_BLOCK_STATE_FREE)
1011 {
1012 pBlockH->dwDataOffset = MOS_ALIGN_CEIL(pBlockH->dwOffsetInStateHeap, dwAlignment);
1013 pBlockH->dwAlignment = pBlockH->dwDataOffset - pBlockH->dwOffsetInStateHeap;
1014 pBlockH->dwDataSize = pBlockH->dwBlockSize - pBlockH->dwAlignment;
1015 pBlockH->pDataPtr = (uint8_t*)pBlockH->pStateHeap->pvLockedHeap + pBlockH->dwDataOffset;
1016
1017 // Free block is now in use - track heap usage
1018 pBlockH->pStateHeap->dwFree -= pBlockL->dwBlockSize;
1019 pBlockH->pStateHeap->dwUsed += pBlockL->dwBlockSize;
1020 }
1021
1022 // Return block object to the pool
1023 ReturnBlockToPool(pBlockL);
1024 }
1025 else
1026 // 2nd block (H) is merged into 1st (L)
1027 {
1028 if (pBlockH->BlockState != MHW_BLOCK_STATE_FREE || // 2nd block must be free
1029 pBlockL->BlockState < MHW_BLOCK_STATE_FREE || // 1nd block must not be in pool
1030 pBlockL->BlockState > MHW_BLOCK_STATE_SUBMITTED) // or deleted
1031 {
1032 return MOS_STATUS_INVALID_PARAMETER;
1033 }
1034
1035 // Merge blocks
1036 pBlockH = DetachBlock(MHW_BLOCK_STATE_FREE, pBlockH);
1037 BLOCK_MANAGER_CHK_NULL(pBlockH);
1038
1039 pBlockL->dwBlockSize += pBlockH->dwBlockSize;
1040 if (pBlockL->BlockState != MHW_BLOCK_STATE_FREE)
1041 {
1042 pBlockL->dwDataSize += pBlockH->dwBlockSize;
1043 pBlockL->pStateHeap->dwFree -= pBlockL->dwBlockSize;
1044 pBlockL->pStateHeap->dwUsed += pBlockL->dwBlockSize;
1045 }
1046
1047 // Add size to the target block list
1048 pList = &(m_BlockList[pBlockL->BlockState]);
1049 pList->dwSize += pBlockH->dwBlockSize;
1050
1051 // Return block object to the pool
1052 ReturnBlockToPool(pBlockH);
1053 }
1054
1055 return eStatus;
1056 }
1057
ResizeBlock(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,uint32_t dwNewSize,uint32_t dwAlignment,bool bBackward)1058 MOS_STATUS MHW_BLOCK_MANAGER::ResizeBlock(
1059 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,
1060 uint32_t dwNewSize,
1061 uint32_t dwAlignment,
1062 bool bBackward) // false => Always grow/shrink forward; true => allow block to grow forward/backwards (moving its start offset)
1063 {
1064 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
1065 PMHW_STATE_HEAP_MEMORY_BLOCK pNewBlock;
1066
1067 BLOCK_MANAGER_CHK_NULL(pBlock);
1068 MHW_ASSERT(dwNewSize > 0);
1069
1070 // Verify block state
1071 if (pBlock->BlockState <= MHW_BLOCK_STATE_POOL || // Cannot touch a block object from pool - invalid data
1072 pBlock->BlockState >= MHW_BLOCK_STATE_DELETED) // Cannot touch a block being deleted
1073 {
1074 return MOS_STATUS_INVALID_PARAMETER;
1075 }
1076
1077 // Shrinking current block
1078 if (dwNewSize < pBlock->dwBlockSize)
1079 {
1080 // Split block into 2 - (bBackwards -> shrink the block by keeping the 2nd half (anchor end of the block)
1081 eStatus = SplitBlockInternal(pBlock, dwNewSize, dwAlignment, bBackward);
1082 if (eStatus != MOS_STATUS_SUCCESS)
1083 {
1084 // This error just means that the block cannot shrink - not an actual issue here
1085 if (eStatus == MOS_STATUS_UNKNOWN)
1086 {
1087 eStatus = MOS_STATUS_SUCCESS;
1088 return eStatus;
1089 }
1090 }
1091
1092 // Select block to be freed
1093 pBlock = (bBackward) ? pBlock->pPrev : pBlock->pNext;
1094 BLOCK_MANAGER_CHK_NULL(pBlock);
1095
1096 if (pBlock->BlockState == MHW_BLOCK_STATE_SUBMITTED)
1097 {
1098 // mark block for release as soon it is no longer in use by GPU
1099 pBlock->bStatic = false;
1100 }
1101 else
1102 {
1103 // Free block - block is in allocated state
1104 FreeBlock(pBlock);
1105 }
1106
1107 // Success!
1108 return MOS_STATUS_SUCCESS;
1109 }
1110
1111 // Check for contiguous available space in forward direction first
1112 uint32_t dwAvailable = pBlock->dwDataSize;
1113 for (pNewBlock = pBlock->pHeapNext;
1114 (dwAvailable < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE);
1115 pNewBlock = pNewBlock->pHeapNext)
1116 {
1117 dwAvailable += pNewBlock->dwBlockSize;
1118 }
1119
1120 // Check for contiguous available space in backward direction
1121 if (bBackward)
1122 {
1123 // Update available space to account for block alignment
1124 dwAvailable += pBlock->dwAlignment - dwAlignment;
1125 for (pNewBlock = pBlock->pHeapPrev;
1126 (dwAvailable < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE);
1127 pNewBlock = pNewBlock->pHeapPrev)
1128 {
1129 dwAvailable += pNewBlock->dwBlockSize;
1130 }
1131 }
1132
1133 // Check if block can be resized
1134 if (dwAvailable < dwNewSize)
1135 {
1136 return MOS_STATUS_UNKNOWN;
1137 }
1138
1139 // Start block expansion forward
1140 for (pNewBlock = pBlock->pHeapNext;
1141 (pBlock->dwDataSize < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE);
1142 pNewBlock = pBlock->pHeapNext)
1143 {
1144 // Next block is too large - split the block before merging
1145 if (pBlock->dwDataSize + pNewBlock->dwBlockSize > dwNewSize)
1146 {
1147 SplitBlockInternal(pNewBlock, dwNewSize - pBlock->dwDataSize, dwAlignment, false);
1148 }
1149
1150 // Merge block with next
1151 MergeBlocksInternal(pBlock, pNewBlock, dwAlignment, false);
1152 }
1153
1154 // Continue expanding backward
1155 if (bBackward)
1156 {
1157 for (pNewBlock = pBlock->pHeapPrev;
1158 (dwAvailable < dwNewSize) && (pNewBlock) && (pNewBlock->BlockState == MHW_BLOCK_STATE_FREE);
1159 pNewBlock = pBlock->pHeapPrev)
1160 {
1161 // Prev block is too large - split the block before merging
1162 uint32_t dwAdjust = MOS_ALIGN_CEIL(pNewBlock->dwOffsetInStateHeap, dwAlignment) - pNewBlock->dwOffsetInStateHeap;
1163 if (pBlock->dwBlockSize + pNewBlock->dwBlockSize - dwAdjust > dwNewSize)
1164 {
1165 SplitBlockInternal(pNewBlock, dwNewSize - pBlock->dwBlockSize, dwAlignment, true);
1166 }
1167
1168 // Merge block with previous
1169 MergeBlocksInternal(pNewBlock, pBlock, dwAlignment, true);
1170 }
1171 }
1172
1173 return eStatus;
1174 }
1175
AllocateWithScratchSpace(uint32_t dwSize,uint32_t dwAlignment,uint32_t dwScratchSpace)1176 PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::AllocateWithScratchSpace(
1177 uint32_t dwSize,
1178 uint32_t dwAlignment,
1179 uint32_t dwScratchSpace)
1180 {
1181 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
1182 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr;
1183 PMHW_STATE_HEAP_MEMORY_BLOCK pScratch = nullptr;
1184
1185 // Fix alignment - must be a power of 2
1186 if (dwAlignment < m_Params.dwHeapGranularity)
1187 {
1188 // Blocks are already aligned
1189 dwAlignment = 1;
1190 }
1191 else
1192 {
1193 dwAlignment--;
1194 dwAlignment |= dwAlignment >> 1;
1195 dwAlignment |= dwAlignment >> 2;
1196 dwAlignment |= dwAlignment >> 4;
1197 dwAlignment |= dwAlignment >> 8;
1198 dwAlignment |= dwAlignment >> 16;
1199 dwAlignment++;
1200 }
1201
1202 // Try to search search state heap with large enough scratch space
1203 // and with enough free space
1204 PMHW_STATE_HEAP pNextStateHeap;
1205 for (PMHW_STATE_HEAP pStateHeap = m_pStateHeap; (pStateHeap); pStateHeap = pNextStateHeap)
1206 {
1207 // Save next state heap
1208 pNextStateHeap = pStateHeap->pNext;
1209
1210 // Space needed for block (accounting for alignment and for scratch space)
1211 uint32_t dwBlockNeeded = MOS_ALIGN_CEIL(dwSize + dwAlignment - 1, m_Params.dwHeapGranularity);
1212
1213 // Scratch space needed = space requested + room for alignment - space already allocated
1214 uint32_t dwScratchNeeded = 0;
1215 if (dwScratchSpace > 0 && pStateHeap->dwScratchSpace < dwScratchSpace)
1216 {
1217 dwScratchNeeded = dwScratchSpace;
1218 if (m_Params.dwHeapGranularity < MHW_SCRATCH_SPACE_ALIGN)
1219 {
1220 dwScratchNeeded += MHW_SCRATCH_SPACE_ALIGN - m_Params.dwHeapGranularity;
1221 }
1222 if (pStateHeap->pScratchSpace)
1223 {
1224 dwScratchNeeded -= pStateHeap->pScratchSpace->dwBlockSize;
1225 }
1226 }
1227
1228 if (pStateHeap->dwSize < dwScratchNeeded)
1229 {
1230 // Heap is too small for current scratch space size - mark for deletion
1231 pStateHeap->pMhwStateHeapInterface->ReleaseStateHeapDyn(pStateHeap);
1232 continue;
1233 }
1234 else if (pStateHeap->dwFree < (dwScratchNeeded + dwBlockNeeded))
1235 {
1236 // Heap can still be used, but currently full, try next heap
1237 continue;
1238 }
1239
1240 // Allocate scratch space first
1241 if (dwScratchNeeded)
1242 {
1243 pScratch = nullptr;
1244
1245 // Already present - EXPAND SCRATCH SPACE
1246 if (pStateHeap->pScratchSpace)
1247 {
1248 // Resize existing scratch space trying to use all free space towards the end of the heap and then growing towards the center.
1249 eStatus = ResizeBlock(pStateHeap->pScratchSpace, dwScratchSpace, MHW_SCRATCH_SPACE_ALIGN, true);
1250 if (eStatus == MOS_STATUS_SUCCESS)
1251 {
1252 pScratch = pStateHeap->pScratchSpace; // Indicates success
1253 pStateHeap->dwScratchSpace = pScratch->dwDataSize; // Available scratch space size (aligned)
1254 }
1255 }
1256
1257 // Scratch space not present or failed to expand - find a new scratch space
1258 if (!pScratch)
1259 {
1260 // Search for scratch space at the end of the heap towards the beginning
1261 // This model allows for better growth without collision with media state heaps
1262 for (pScratch = pStateHeap->pMemoryTail; pScratch != pBlock; pScratch = pScratch->pHeapPrev)
1263 {
1264 if (pScratch->BlockState == MHW_BLOCK_STATE_FREE &&
1265 pScratch->dwBlockSize >= dwScratchNeeded)
1266 {
1267 // Found scratch space large enough
1268 break;
1269 }
1270 }
1271
1272 // Not enough contiguous space for scratch in heap - try next heap
1273 if (!pScratch)
1274 {
1275 continue;
1276 }
1277
1278 // CREATE SCRATCH SPACE
1279
1280 // Split block to the necessary size, using the higher portion (closer to end of the heap)
1281 // NOTE: Block select could be much larger than needed, even the size of the entire state heap - that's why it needs to be split
1282 eStatus = SplitBlockInternal(pScratch, dwScratchNeeded, MHW_SCRATCH_SPACE_ALIGN, true);
1283 if (eStatus == MOS_STATUS_UNKNOWN) eStatus = MOS_STATUS_SUCCESS; // Don't care if block could not be split
1284 if (eStatus != MOS_STATUS_SUCCESS)
1285 {
1286 continue;
1287 }
1288
1289 // Move block to allocated list, mark it as static (do not release upon completion)
1290 AllocateBlockInternal(pScratch, MHW_SCRATCH_SPACE_ALIGN);
1291 pScratch->bStatic = true;
1292
1293 // Free the old scratch space
1294 // NOTE: scratch spaces are also tracked by Sync Tags and maintained in submitted/allocated lists,
1295 // If in use, it just clears the bStatic flag, so it will be freed when no longer in use.
1296 if (pStateHeap->pScratchSpace)
1297 {
1298 FreeBlock(pStateHeap->pScratchSpace);
1299 }
1300
1301 // Setup new heap scratch space and size
1302 pStateHeap->pScratchSpace = pScratch;
1303 pStateHeap->dwScratchSpace = pScratch->dwDataSize;
1304 }
1305 }
1306
1307 // Try to allocate block in the same heap as the scratch space
1308 pBlock = AllocateBlock(dwSize, dwAlignment, pStateHeap);
1309 if (pBlock)
1310 {
1311 break;
1312 }
1313 }
1314
1315 return pBlock;
1316 }
1317
AllocateBlock(uint32_t dwSize,uint32_t dwAlignment,PMHW_STATE_HEAP pHeapAffinity)1318 PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::AllocateBlock(
1319 uint32_t dwSize,
1320 uint32_t dwAlignment,
1321 PMHW_STATE_HEAP pHeapAffinity)
1322 {
1323 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = nullptr;
1324 PMHW_BLOCK_LIST pFree = &m_BlockList[MHW_BLOCK_STATE_FREE];
1325 uint32_t dwAdjust; // Offset adjustment for alignment purposes
1326 uint32_t dwAllocSize; // Actual allocation size accounting for alignment and other restrictions
1327 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
1328
1329 // Fix alignment - must be a power of 2 (minimum 1)
1330 if (dwAlignment) dwAlignment--;
1331 dwAlignment |= dwAlignment >> 1;
1332 dwAlignment |= dwAlignment >> 2;
1333 dwAlignment |= dwAlignment >> 4;
1334 dwAlignment |= dwAlignment >> 8;
1335 dwAlignment |= dwAlignment >> 16;
1336 dwAlignment++;
1337
1338 // Search must include space for block granularity
1339 if (dwAlignment <= m_Params.dwHeapGranularity)
1340 {
1341 // Alignment should be fulfilled by the heap granularity
1342 dwAllocSize = dwSize;
1343 }
1344 else
1345 {
1346 // Worst case scenario - original implementation was checking alignment of
1347 // each free block, but it was overkill - so now we just consider the worst case
1348 dwAllocSize = dwSize + dwAlignment - m_Params.dwHeapGranularity;
1349 }
1350
1351 // Enforce min block size
1352 dwAllocSize = MOS_MAX(m_Params.dwHeapBlockMinSize, dwAllocSize);
1353
1354 // Search list of free blocks for the first match
1355 for (pBlock = pFree->pHead; pBlock != nullptr; pBlock = pBlock->pNext)
1356 {
1357 // Skip this heap if we are looking for allocation in a specific heap
1358 if (pHeapAffinity && pBlock->pStateHeap != pHeapAffinity)
1359 {
1360 continue;
1361 }
1362
1363 // Check if aligned block fits the request -> break with a successful block
1364 if (pBlock->dwBlockSize >= dwAllocSize)
1365 {
1366 break;
1367 }
1368 }
1369
1370 // No block was found - fail search
1371 if (!pBlock)
1372 {
1373 return nullptr;
1374 }
1375
1376 // Block was found, adjust the allocation size to account for
1377 // heap granularity and block alignment
1378 dwAdjust = MOS_ALIGN_OFFSET(pBlock->dwOffsetInStateHeap, dwAlignment); // Increase in size to align data
1379 dwAllocSize = MOS_ALIGN_CEIL(dwSize + dwAdjust, m_Params.dwHeapGranularity); // Account for heap granularity (avoid odd addresses in heap)
1380 dwAllocSize = MOS_MAX(dwAllocSize, m_Params.dwHeapBlockMinSize);
1381
1382 // Just a precaution - sanity check - in case of last block in heap, and total heap size is not a multiple of granularity
1383 if (pBlock->dwBlockSize < dwAllocSize)
1384 {
1385 // This should never happend because it is part of the search condition!
1386 MHW_ASSERT(pBlock->dwBlockSize >= (dwAdjust + dwSize));
1387
1388 dwAllocSize = pBlock->dwBlockSize;
1389 }
1390
1391 // Split block, move to allocated list
1392 if (pBlock->dwBlockSize > dwAllocSize)
1393 {
1394 // Split free block in 2, keep the first part (lower offset)
1395 eStatus = SplitBlockInternal(pBlock, dwAllocSize, dwAlignment, false);
1396 if (eStatus != MOS_STATUS_SUCCESS &&
1397 eStatus != MOS_STATUS_UNKNOWN)
1398 {
1399 MHW_ASSERTMESSAGE("ERROR: AllocateBlock: Failed to allocate block");
1400 return nullptr;
1401 }
1402 }
1403
1404 // Move block from free to allocated queue
1405 DetachBlock(MHW_BLOCK_STATE_FREE, pBlock);
1406 AttachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlock, MHW_BLOCK_POSITION_TAIL);
1407 pBlock->pStateHeap->dwUsed += pBlock->dwBlockSize;
1408 pBlock->pStateHeap->dwFree -= pBlock->dwBlockSize;
1409
1410 // Reset some fields
1411 pBlock->bDelete = false;
1412 FrameTrackerTokenFlat_Validate(&pBlock->trackerToken);
1413
1414 // Setup aligned offset, size and data
1415 pBlock->dwDataOffset = MOS_ALIGN_CEIL(pBlock->dwOffsetInStateHeap, dwAlignment);
1416 pBlock->dwAlignment = pBlock->dwDataOffset - pBlock->dwOffsetInStateHeap;
1417 pBlock->dwDataSize = pBlock->dwBlockSize - pBlock->dwAlignment;
1418 pBlock->pDataPtr = (uint8_t*)pBlock->pStateHeap->pvLockedHeap + pBlock->dwDataOffset;
1419
1420 // return block to client
1421 return pBlock;
1422 }
1423
FreeBlock(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)1424 MOS_STATUS MHW_BLOCK_MANAGER::FreeBlock(
1425 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock)
1426 {
1427 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
1428
1429 BLOCK_MANAGER_CHK_NULL(pBlock);
1430
1431 // Block still in use - mark for auto-release when complete
1432 if (pBlock->BlockState == MHW_BLOCK_STATE_SUBMITTED)
1433 {
1434 // sync tag not provided or block still in use - flag it for automatic release when complete
1435 if (!FrameTrackerTokenFlat_IsExpired(&pBlock->trackerToken))
1436 {
1437 pBlock->bStatic = false;
1438 return eStatus;
1439 }
1440 }
1441 else if (pBlock->BlockState != MHW_BLOCK_STATE_ALLOCATED)
1442 {
1443 return MOS_STATUS_INVALID_PARAMETER;
1444 }
1445
1446 // Remove block from its current list
1447 DetachBlock(pBlock->BlockState, pBlock);
1448
1449 // If block is marked for deletion - move to deleted list so it cannot be reallocated
1450 if (pBlock->bDelete)
1451 {
1452 MHW_STATE_HEAP *pStateHeap = pBlock->pStateHeap;
1453 pStateHeap->dwUsed -= pBlock->dwBlockSize;
1454
1455 AttachBlock(MHW_BLOCK_STATE_DELETED, pBlock, MHW_BLOCK_POSITION_TAIL);
1456
1457 // Last block was removed from StateHeap -> state heap may be unregistered and deleted
1458 if (pStateHeap->dwUsed == 0)
1459 {
1460 pStateHeap->pMhwStateHeapInterface->ReleaseStateHeapDyn(pStateHeap);
1461 }
1462 }
1463 else
1464 {
1465 // Update state heap usage
1466 pBlock->pStateHeap->dwUsed -= pBlock->dwBlockSize;
1467 pBlock->pStateHeap->dwFree += pBlock->dwBlockSize;
1468
1469 // Blocks are freed and placed at the beginning of the free block list
1470 AttachBlock(MHW_BLOCK_STATE_FREE, pBlock, MHW_BLOCK_POSITION_TAIL);
1471
1472 // Consolidate memory immediately after release - so free block are ALWAYS merged
1473 ConsolidateBlock(pBlock);
1474 }
1475
1476 return eStatus;
1477 }
1478
CalculateSpaceNeeded(const uint32_t * pdwSizes,int32_t iCount,uint32_t dwAlignment,bool bHeapAffinity,PMHW_STATE_HEAP pHeapAffinity)1479 uint32_t MHW_BLOCK_MANAGER::CalculateSpaceNeeded(
1480 const uint32_t *pdwSizes,
1481 int32_t iCount,
1482 uint32_t dwAlignment,
1483 bool bHeapAffinity,
1484 PMHW_STATE_HEAP pHeapAffinity)
1485 {
1486 uint32_t dwNeeded = 0;
1487 uint8_t SortedIndex[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY];
1488 uint32_t FreeBlockSizes[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY * 2];
1489 uint32_t dwBlockOverhead = 0; // Block size overhead to ensure alignment
1490 uint32_t dwAllocSize;
1491
1492 // Check parameters
1493 if (iCount <= 0 || iCount > MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY)
1494 {
1495 return dwNeeded;
1496 }
1497
1498 // This is the minimum block
1499 uint32_t dwBlockGranularity = m_Params.dwHeapGranularity;
1500 uint32_t dwBlockMinSize = m_Params.dwHeapBlockMinSize;
1501 if (dwAlignment > dwBlockGranularity)
1502 {
1503 dwBlockOverhead = dwAlignment - dwBlockGranularity;
1504 }
1505
1506 // Very simple case - single block search
1507 if (iCount == 1)
1508 {
1509 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = m_BlockList[MHW_BLOCK_STATE_FREE].pHead;
1510 dwNeeded = pdwSizes[0];
1511 for ( ; pBlock != nullptr && dwNeeded > 0; pBlock = pBlock->pNext)
1512 {
1513 if (bHeapAffinity && pHeapAffinity != pBlock->pStateHeap)
1514 {
1515 continue;
1516 }
1517
1518 if (dwNeeded < pBlock->dwBlockSize)
1519 {
1520 dwNeeded = 0;
1521 }
1522 }
1523
1524 return dwNeeded;
1525 }
1526
1527 // Sort input block sizes (with index, to avoid modifying the input array)
1528 Mhw_BlockManager_ReverseMergeSort_With_Index(pdwSizes, iCount, SortedIndex);
1529
1530 // Read all available blocks, keep the largest blocks
1531 int iIndex = 0;
1532 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock = m_BlockList[MHW_BLOCK_STATE_FREE].pHead;
1533 for ( ; pBlock != nullptr; pBlock = pBlock->pNext)
1534 {
1535 // Select free blocks that fit the request
1536 if (bHeapAffinity && pHeapAffinity != pBlock->pStateHeap) continue;
1537 FreeBlockSizes[iIndex++] = pBlock->dwBlockSize;
1538
1539 // If buffer is full, sort it, only take the largest blocks (overwrite remaining blocks)
1540 if (iIndex == MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY * 2)
1541 {
1542 Mhw_BlockManager_ReverseMergeSort(FreeBlockSizes, iIndex);
1543 iIndex = iCount;
1544 }
1545 }
1546
1547 // Final sort after all block are in
1548 Mhw_BlockManager_ReverseMergeSort(FreeBlockSizes, iIndex);
1549 FreeBlockSizes[iIndex] = 0; // Null termination simplifies the algorithm that follows
1550
1551 // Start process of fitting requested blocks with available blocks
1552 uint32_t *pFreeBlock = &FreeBlockSizes[0];
1553 for (iIndex = 0; iIndex < iCount; iIndex++)
1554 {
1555 dwAllocSize = MOS_ALIGN_CEIL(pdwSizes[SortedIndex[iIndex]] + dwBlockOverhead, dwBlockGranularity);
1556 dwAllocSize = MOS_MAX(dwAllocSize, dwBlockMinSize);
1557
1558 // Block doesn't fit - add to space needed
1559 if (dwAllocSize > *pFreeBlock)
1560 {
1561 dwNeeded += dwAllocSize;
1562 }
1563 else
1564 {
1565 // Allocate block
1566 uint32_t dwRemaining = *pFreeBlock = *pFreeBlock - dwAllocSize;
1567 uint32_t *pAux;
1568
1569 // Remaining is out of order, keep list sorted using insertion sort
1570 if (dwRemaining < pFreeBlock[1])
1571 {
1572 for (pAux = pFreeBlock; dwRemaining < pAux[1]; pAux++) pAux[0] = pAux[1];
1573 pAux[0] = dwRemaining;
1574 }
1575 }
1576 }
1577
1578 return dwNeeded;
1579 }
1580
SubmitBlock(PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,const FrameTrackerTokenFlat * trackerToken)1581 MOS_STATUS MHW_BLOCK_MANAGER::SubmitBlock(
1582 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock,
1583 const FrameTrackerTokenFlat *trackerToken)
1584 {
1585 MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
1586
1587 BLOCK_MANAGER_CHK_NULL(pBlock);
1588
1589 if (pBlock->BlockState == MHW_BLOCK_STATE_POOL || // Block cannot be submitted directly from pool
1590 pBlock->BlockState == MHW_BLOCK_STATE_FREE || // or from free (must be allocated first)
1591 pBlock->BlockState == MHW_BLOCK_STATE_DELETED) // or from deleted (the block is being DELETED for crying out loud!)
1592 {
1593 MHW_ASSERTMESSAGE("ERROR: SubmitBlock: Block in invalid state, cannot be enqueued");
1594 return MOS_STATUS_UNKNOWN;
1595 }
1596
1597 // Detach block from whatever list it's in - could be in SUBMITTED state (block is being reused - i.e., kernel or scratch space)
1598 pBlock = DetachBlock(pBlock->BlockState, pBlock);
1599 BLOCK_MANAGER_CHK_NULL(pBlock);
1600
1601 // Set block sync tag - used for refreshing the block status
1602 FrameTrackerTokenFlat_Merge(&pBlock->trackerToken, trackerToken);
1603
1604 BLOCK_MANAGER_CHK_STATUS(AttachBlock(MHW_BLOCK_STATE_SUBMITTED, pBlock, MHW_BLOCK_POSITION_TAIL));
1605
1606 return eStatus;
1607 }
1608
AllocateMultiple(uint32_t * pdwSizes,int32_t iCount,uint32_t dwAlignment,bool bHeapAffinity,PMHW_STATE_HEAP pHeapAffinity)1609 PMHW_STATE_HEAP_MEMORY_BLOCK MHW_BLOCK_MANAGER::AllocateMultiple(
1610 uint32_t *pdwSizes,
1611 int32_t iCount,
1612 uint32_t dwAlignment,
1613 bool bHeapAffinity,
1614 PMHW_STATE_HEAP pHeapAffinity)
1615 {
1616 PMHW_STATE_HEAP pStateHeap;
1617 uint32_t dwTotalSize = 0;
1618 uint8_t SortedIndex[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // uint8_t is only used because the array size is <256
1619 PMHW_STATE_HEAP_MEMORY_BLOCK pBlockArray[MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY]; // This could be allocated dynamically, same for the above
1620 PMHW_STATE_HEAP_MEMORY_BLOCK pBlock;
1621
1622 // Clear the result
1623 pBlockArray[0] = nullptr;
1624
1625 if (iCount <= 0 || iCount > MHW_BLOCK_MANAGER_MAX_BLOCK_ARRAY)
1626 {
1627 return nullptr;
1628 }
1629
1630 // Get total allocation size
1631 for (int i = 0; i < iCount; i++)
1632 {
1633 dwTotalSize += pdwSizes[i];
1634 }
1635
1636 Mhw_BlockManager_ReverseMergeSort_With_Index(pdwSizes, iCount, SortedIndex);
1637
1638 if (bHeapAffinity)
1639 {
1640 if (pHeapAffinity)
1641 {
1642 // Set heap affinity - all blocks will be allocated from this one particular heap
1643 pStateHeap = pHeapAffinity;
1644 }
1645 else
1646 {
1647 // Heap affinity is not set - start from the current active heap (most recently allocated heap)
1648 pStateHeap = m_pStateHeap;
1649 }
1650 }
1651 else
1652 { // Don't care about heap affinity, blocks can be spread across all available state heaps
1653 // Just calculate total space available to make sure we have it
1654 uint32_t dwTotalHeapSpace = 0;
1655 for (pStateHeap = m_pStateHeap; pStateHeap != nullptr; pStateHeap = pStateHeap->pNext)
1656 {
1657 // Calculate total free space in all available heaps
1658 // NOTE: Heap being deleted as free space set to 0
1659 // and all Free blocks are now Deleted (in deleted queue)
1660 dwTotalHeapSpace += pStateHeap->dwFree;
1661 }
1662
1663 // Not enough space
1664 if (dwTotalHeapSpace < dwTotalSize)
1665 {
1666 return pBlockArray[0];
1667 }
1668 }
1669
1670 // Loop to try loading all kernels into the same heap
1671 do {
1672 if ( (!bHeapAffinity) || // no affinity set -> blocks can be spread across multiple heaps
1673 pStateHeap->dwFree >= dwTotalSize) // this heap has enough free space
1674 {
1675 int32_t i, j;
1676
1677 // Allocate array according to size
1678 for (i = 0, pBlock = nullptr; i < iCount; i++)
1679 { // NOTE - don't care about scratch space, already checked
1680 pBlock = pBlockArray[SortedIndex[i]] = AllocateBlock(pdwSizes[SortedIndex[i]], dwAlignment, pStateHeap);
1681 if (!pBlock)
1682 {
1683 break;
1684 }
1685 }
1686
1687 // Allocations all sucessfull, quit search
1688 if (pBlock)
1689 {
1690 break;
1691 }
1692
1693 // One of the block allocations failed - free all blocks already allocated (try another heap)
1694 for (j = 0; j < i; j++)
1695 {
1696 FreeBlock(pBlockArray[SortedIndex[j]]);
1697 }
1698
1699 // Clear the output
1700 pBlockArray[0] = nullptr;
1701 }
1702
1703 // Try another heap if bHeapAffinity is set
1704 if (bHeapAffinity)
1705 {
1706 pStateHeap = (pHeapAffinity) ? nullptr : pStateHeap->pNext;
1707 }
1708 } while (pStateHeap);
1709
1710 // Allocation successful, reorder blocks according to original request
1711 if (pBlockArray[0])
1712 {
1713 for (int32_t i = 0; i < iCount; i++)
1714 {
1715 pBlock = DetachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlockArray[i]);
1716 AttachBlock(MHW_BLOCK_STATE_ALLOCATED, pBlock, MHW_BLOCK_POSITION_TAIL);
1717 }
1718 }
1719
1720 return pBlockArray[0];
1721 }
1722