1 /*
2 * Copyright (c) 2017, Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22 //!
23 //! \file     memory_block_manager.cpp
24 //! \brief    Implements functionalities pertaining to the memory block manager
25 //!
26 
27 #include "memory_block_manager.h"
28 
~MemoryBlockManager()29 MemoryBlockManager::~MemoryBlockManager()
30 {
31     HEAP_FUNCTION_ENTER;
32     m_heaps.clear();
33     m_deletedHeaps.clear();
34     HEAP_VERBOSEMESSAGE("Total heap size is %d", m_totalSizeOfHeaps);
35     for (uint8_t i = 0; i < MemoryBlockInternal::State::stateCount; i++)
36     {
37         HEAP_VERBOSEMESSAGE("%d blocks in pool %d with size %d", m_sortedBlockListNumEntries[i], i, m_sortedBlockListSizes[i]);
38 
39         // pool blocks are not part of the adjacency list and must be freed separately
40         if (i == MemoryBlockInternal::State::pool)
41         {
42             auto curr = m_sortedBlockList[i];
43             MemoryBlockInternal *next = nullptr;
44             while (curr != nullptr)
45             {
46                 next = curr->m_stateNext;
47                 MOS_Delete(curr);
48                 curr = next;
49             }
50         }
51     }
52 }
53 
AcquireSpace(AcquireParams & params,std::vector<MemoryBlock> & blocks,uint32_t & spaceNeeded)54 MOS_STATUS MemoryBlockManager::AcquireSpace(
55     AcquireParams &params,
56     std::vector<MemoryBlock> &blocks,
57     uint32_t &spaceNeeded)
58 {
59     HEAP_FUNCTION_ENTER;
60 
61     if (params.m_blockSizes.empty())
62     {
63         HEAP_ASSERTMESSAGE("No space is being requested");
64         return MOS_STATUS_INVALID_PARAMETER;
65     }
66 
67     if (m_sortedSizes.size() != params.m_blockSizes.size())
68     {
69         m_sortedSizes.resize(params.m_blockSizes.size());
70     }
71     uint32_t alignment = MOS_MAX(m_blockAlignment, MOS_ALIGN_CEIL(params.m_alignment, m_blockAlignment));
72     uint32_t idx = 0;
73     auto sortedIterator = m_sortedSizes.begin();
74     for (auto requestIterator = params.m_blockSizes.begin();
75         requestIterator != params.m_blockSizes.end();
76         ++requestIterator)
77     {
78         if (sortedIterator == m_sortedSizes.end())
79         {
80             HEAP_ASSERTMESSAGE("sortedSizes is expected to be the same size as the number of blocks requested");
81             return MOS_STATUS_UNKNOWN;
82         }
83         (*sortedIterator).m_originalIdx = idx++;
84         (*sortedIterator).m_blockSize = MOS_ALIGN_CEIL(*requestIterator, alignment);
85         ++sortedIterator;
86     }
87     if (m_sortedSizes.size() > 1)
88     {
89         m_sortedSizes.sort([](SortedSizePair &a, SortedSizePair &b) { return a.m_blockSize > b.m_blockSize; });
90     }
91 
92     if (m_sortedBlockListNumEntries[MemoryBlockInternal::submitted] > m_numSubmissionsForRefresh)
93     {
94         bool blocksUpdated = false;
95         HEAP_CHK_STATUS(RefreshBlockStates(blocksUpdated));
96     }
97 
98     spaceNeeded = 0;
99     HEAP_CHK_STATUS(IsSpaceAvailable(params, spaceNeeded));
100     if (spaceNeeded == 0)
101     {
102         HEAP_CHK_STATUS(AllocateSpace(params, blocks));
103         return MOS_STATUS_SUCCESS;
104     }
105 
106     blocks.clear();
107     return MOS_STATUS_CLIENT_AR_NO_SPACE;
108 }
109 
SubmitBlocks(std::vector<MemoryBlock> & blocks)110 MOS_STATUS MemoryBlockManager::SubmitBlocks(std::vector<MemoryBlock> &blocks)
111 {
112     HEAP_FUNCTION_ENTER;
113 
114     if (blocks.empty())
115     {
116         HEAP_ASSERTMESSAGE("No blocks to submit");
117         return MOS_STATUS_INVALID_PARAMETER;
118     }
119 
120     for (uint32_t i = 0; i < blocks.size(); ++i)
121     {
122         if (!blocks[i].IsValid())
123         {
124             HEAP_ASSERTMESSAGE("Block %d is not valid and may not be submitted", i);
125             return MOS_STATUS_INVALID_PARAMETER;
126         }
127 
128         auto internalBlock = blocks[i].GetInternalBlock();
129         HEAP_CHK_NULL(internalBlock);
130         HEAP_CHK_STATUS(RemoveBlockFromSortedList(internalBlock, internalBlock->GetState()));
131         HEAP_CHK_STATUS(internalBlock->Submit());
132         HEAP_CHK_STATUS(AddBlockToSortedList(internalBlock, internalBlock->GetState()));
133     }
134 
135     return MOS_STATUS_SUCCESS;
136 }
137 
ClearSpace(MemoryBlock & block)138 MOS_STATUS MemoryBlockManager::ClearSpace(MemoryBlock &block)
139 {
140     HEAP_FUNCTION_ENTER;
141 
142     if (!block.IsValid())
143     {
144         HEAP_ASSERTMESSAGE("Block is not valid and may not be cleared");
145         return MOS_STATUS_INVALID_PARAMETER;
146     }
147 
148     auto internalBlock = block.GetInternalBlock();
149     HEAP_CHK_NULL(internalBlock);
150     internalBlock->ClearStatic();
151     bool blocksUpdated = false;
152     HEAP_CHK_STATUS(RefreshBlockStates(blocksUpdated));
153     // Reset memory block to blank state to return to client
154     block = MemoryBlock();
155 
156     return MOS_STATUS_SUCCESS;
157 }
158 
RefreshBlockStates(bool & blocksUpdated)159 MOS_STATUS MemoryBlockManager::RefreshBlockStates(bool &blocksUpdated)
160 {
161     HEAP_FUNCTION_ENTER_VERBOSE;
162 
163     if ((!m_useProducer && m_trackerData == nullptr)
164         || (m_useProducer && m_trackerProducer == nullptr))
165     {
166         HEAP_ASSERTMESSAGE("It is not possible to check current tracker ID");
167         return MOS_STATUS_INVALID_PARAMETER;
168     }
169 
170     blocksUpdated = false;
171     uint32_t currTrackerId = 0;
172     if (!m_useProducer)
173     {
174         currTrackerId = *m_trackerData;
175     }
176 
177     auto block = m_sortedBlockList[MemoryBlockInternal::State::submitted];
178     MemoryBlockInternal *nextSubmitted = nullptr;
179     while (block != nullptr)
180     {
181         nextSubmitted = block->m_stateNext;
182         FrameTrackerToken *trackerToken = block->GetTrackerToken();
183         if ( (!m_useProducer && block->GetTrackerId() <= currTrackerId)
184             ||(m_useProducer && trackerToken->IsExpired()))
185         {
186             auto heap = block->GetHeap();
187             HEAP_CHK_NULL(heap);
188 
189             if (heap->IsFreeInProgress())
190             {
191                 // Add the block to deleted list instead of freed to prevent it from being reused
192                 HEAP_CHK_STATUS(RemoveBlockFromSortedList(block, block->GetState()));
193                 HEAP_CHK_STATUS(block->Delete());
194                 HEAP_CHK_STATUS(AddBlockToSortedList(block, block->GetState()));
195                 block = nextSubmitted;
196                 continue;
197             }
198 
199             HEAP_CHK_STATUS(RemoveBlockFromSortedList(block, block->GetState()));
200             HEAP_CHK_STATUS(block->Free());
201             HEAP_CHK_STATUS(AddBlockToSortedList(block, block->GetState()));
202 
203             // Consolidate free blocks
204             auto prev = block->GetPrev(), next = block->GetNext();
205             if (prev && prev->GetState() == MemoryBlockInternal::State::free)
206             {
207                 HEAP_CHK_STATUS(MergeBlocks(prev, block));
208                 // re-assign block to pPrev for use in MergeBlocks with pNext
209                 block = prev;
210             }
211             else if (prev == nullptr)
212             {
213                 HEAP_ASSERTMESSAGE("The previous block should always be valid");
214                 return MOS_STATUS_UNKNOWN;
215             }
216 
217             if (next && next->GetState() == MemoryBlockInternal::State::free)
218             {
219                 HEAP_CHK_STATUS(MergeBlocks(block, next));
220             }
221 
222             blocksUpdated = true;
223         }
224         block = nextSubmitted;
225     }
226 
227     if (blocksUpdated && !m_deletedHeaps.empty())
228     {
229         HEAP_CHK_STATUS(CompleteHeapDeletion());
230     }
231 
232     return MOS_STATUS_SUCCESS;
233 }
234 
RegisterHeap(uint32_t heapId,uint32_t size,bool hwWriteOnly)235 MOS_STATUS MemoryBlockManager::RegisterHeap(uint32_t heapId, uint32_t size , bool hwWriteOnly)
236 {
237     HEAP_FUNCTION_ENTER;
238 
239     MOS_STATUS eStatus = MOS_STATUS_SUCCESS;
240 
241     auto heap = MOS_New(Heap, heapId);
242     HEAP_CHK_NULL(heap);
243     eStatus = heap->RegisterOsInterface(m_osInterface);
244     if (MOS_FAILED(eStatus))
245     {
246         MOS_Delete(heap);
247         HEAP_CHK_STATUS(eStatus);
248     }
249     size = MOS_ALIGN_CEIL(size, m_heapAlignment);
250 
251     if (hwWriteOnly)
252     {
253         heap->SetHeapHwWriteOnly(hwWriteOnly);
254     }
255 
256     eStatus = heap->Allocate(size, m_lockHeapsOnAllocate);
257     if (MOS_FAILED(eStatus))
258     {
259         MOS_Delete(heap);
260         HEAP_CHK_STATUS(eStatus);
261     }
262 
263     if (heap->IsValid())
264     {
265         MemoryBlockInternal *adjacencyListBegin = nullptr;
266         adjacencyListBegin = MOS_New(MemoryBlockInternal);
267         if (adjacencyListBegin == nullptr)
268         {
269             MOS_Delete(heap);
270             HEAP_CHK_STATUS(MOS_STATUS_NULL_POINTER);
271         }
272         auto block = GetBlockFromPool();
273         if (block == nullptr)
274         {
275             MOS_Delete(adjacencyListBegin);
276             MOS_Delete(heap);
277             HEAP_ASSERTMESSAGE("block be null");
278             return MOS_STATUS_NULL_POINTER;
279         }
280 
281         std::shared_ptr<HeapWithAdjacencyBlockList> managedHeap = nullptr;
282         managedHeap = MakeShared<HeapWithAdjacencyBlockList>();
283         if (managedHeap == nullptr)
284         {
285             MOS_Delete(heap);
286             MOS_Delete(adjacencyListBegin);
287             HEAP_CHK_STATUS(MOS_STATUS_NULL_POINTER);
288         }
289         managedHeap->m_heap = heap;
290         managedHeap->m_size = managedHeap->m_heap->GetSize();
291         managedHeap->m_adjacencyListBegin = adjacencyListBegin;
292         m_totalSizeOfHeaps += managedHeap->m_size;
293         m_heaps.push_back(std::move(managedHeap));
294 
295         HEAP_CHK_STATUS(block->Create(
296             heap,
297             MemoryBlockInternal::State::free,
298             adjacencyListBegin,
299             0,
300             size,
301             MemoryBlockInternal::m_invalidTrackerId));
302         HEAP_CHK_STATUS(AddBlockToSortedList(block, block->GetState()));
303     }
304     else
305     {
306         HEAP_ASSERTMESSAGE("Cannot register invalid heap");
307         return MOS_STATUS_INVALID_PARAMETER;
308     }
309 
310     return eStatus;
311 }
312 
UnregisterHeap(uint32_t heapId)313 MOS_STATUS MemoryBlockManager::UnregisterHeap(uint32_t heapId)
314 {
315     HEAP_FUNCTION_ENTER;
316 
317     for (auto iterator = m_heaps.begin(); iterator != m_heaps.end(); ++iterator)
318     {
319         if (heapId == (*iterator)->m_heap->GetId())
320         {
321             bool blocksUpdated = false;
322             HEAP_CHK_STATUS(RefreshBlockStates(blocksUpdated));
323             (*iterator)->m_heap->PrepareForFree();
324             m_totalSizeOfHeaps -= (*iterator)->m_heap->GetSize();
325 
326             // free blocks may be removed right away
327             auto block = m_sortedBlockList[MemoryBlockInternal::State::free];
328             MemoryBlockInternal *next = nullptr;
329             while (block != nullptr)
330             {
331                 next = block->m_stateNext;
332                 auto heap = block->GetHeap();
333                 if (heap != nullptr)
334                 {
335                     if (heap->GetId() == heapId)
336                     {
337                         HEAP_CHK_STATUS(RemoveBlockFromSortedList(block, block->GetState()));
338                         HEAP_CHK_STATUS(block->Delete());
339                         HEAP_CHK_STATUS(AddBlockToSortedList(block, block->GetState()));
340                     }
341                 }
342                 else
343                 {
344                     HEAP_ASSERTMESSAGE("A block with an invalid heap is in the free list!");
345                     return MOS_STATUS_UNKNOWN;
346                 }
347                 block = next;
348             }
349 
350             m_deletedHeaps.push_back((*iterator));
351             m_heaps.erase(iterator);
352             HEAP_CHK_STATUS(CompleteHeapDeletion());
353             return MOS_STATUS_SUCCESS;
354         }
355     }
356 
357     return MOS_STATUS_SUCCESS;
358 }
359 
CompleteHeapDeletion()360 MOS_STATUS MemoryBlockManager::CompleteHeapDeletion()
361 {
362     HEAP_FUNCTION_ENTER_VERBOSE;
363 
364     auto iterator = m_deletedHeaps.begin();
365     while (iterator != m_deletedHeaps.end())
366     {
367         // if the heap is still not empty, continue with the loop
368         if ((*iterator)->m_heap->GetUsedSize() == 0)
369         {
370             uint32_t heapId = (*iterator)->m_heap->GetId();
371             HEAP_CHK_STATUS(RemoveHeapFromSortedBlockList(heapId));
372             iterator = m_deletedHeaps.erase(iterator);
373         }
374         else
375         {
376             ++iterator;
377         }
378     }
379 
380     return MOS_STATUS_SUCCESS;
381 }
382 
RegisterTrackerResource(uint32_t * trackerData)383 MOS_STATUS MemoryBlockManager::RegisterTrackerResource(uint32_t *trackerData)
384 {
385     HEAP_FUNCTION_ENTER;
386     HEAP_CHK_NULL(trackerData);
387     m_trackerData = trackerData;
388     return MOS_STATUS_SUCCESS;
389 }
390 
RegisterTrackerProducer(FrameTrackerProducer * trackerProducer)391 MOS_STATUS MemoryBlockManager::RegisterTrackerProducer(FrameTrackerProducer *trackerProducer)
392 {
393     HEAP_FUNCTION_ENTER;
394     HEAP_CHK_NULL(trackerProducer);
395     m_trackerProducer = trackerProducer;
396     m_useProducer = true;
397     return MOS_STATUS_SUCCESS;
398 }
399 
RegisterOsInterface(PMOS_INTERFACE osInterface)400 MOS_STATUS MemoryBlockManager::RegisterOsInterface(PMOS_INTERFACE osInterface)
401 {
402     HEAP_FUNCTION_ENTER;
403     HEAP_CHK_NULL(osInterface);
404     m_osInterface = osInterface;
405     return MOS_STATUS_SUCCESS;
406 }
407 
IsSpaceAvailable(AcquireParams & params,uint32_t & spaceNeeded)408 MOS_STATUS MemoryBlockManager::IsSpaceAvailable(
409     AcquireParams &params,
410     uint32_t &spaceNeeded)
411 {
412     HEAP_FUNCTION_ENTER_VERBOSE;
413 
414     if (m_sortedSizes.empty())
415     {
416         HEAP_ASSERTMESSAGE("No space is being requested");
417         return MOS_STATUS_INVALID_PARAMETER;
418     }
419     if (m_sortedBlockList[MemoryBlockInternal::State::free] == nullptr)
420     {
421         bool blocksUpdated = false;
422         HEAP_CHK_STATUS(RefreshBlockStates(blocksUpdated));
423         if (!blocksUpdated)
424         {
425             HEAP_NORMALMESSAGE("All heap space is in use by active workloads.");
426         }
427     }
428 
429     auto block = m_sortedBlockList[MemoryBlockInternal::State::free];
430 
431     for (auto requestIterator = m_sortedSizes.begin();
432         requestIterator != m_sortedSizes.end();
433         ++requestIterator)
434     {
435         if (block == nullptr)
436         {
437             // There is no free space available, add up all requested sizes
438             spaceNeeded += (*requestIterator).m_blockSize;
439         }
440         else
441         {
442             // Determine whether or not there is space available
443             if ((*requestIterator).m_blockSize > block->GetSize())
444             {
445                 // The requested size is larger than the largest free block size
446                 spaceNeeded += (*requestIterator).m_blockSize;
447                 continue;
448             }
449             else
450             {
451                 // determine if there is enough space in the heaps
452                 auto freeBlockSize = block->GetSize();
453                 while (requestIterator != m_sortedSizes.end())
454                 {
455                     if ((*requestIterator).m_blockSize < freeBlockSize)
456                     {
457                         // if the aligned size fits within the current free block, consider it used
458                         freeBlockSize -= (*requestIterator).m_blockSize;
459                         ++requestIterator;
460                     }
461                     else
462                     {
463                         // if the aligned size does not fit within the curret free block, continue for loop
464                         break;
465                     }
466                 }
467                 if (requestIterator == m_sortedSizes.end())
468                 {
469                     break;
470                 }
471             }
472         }
473         if (block != nullptr)
474         {
475             block = block->m_stateNext;
476         }
477     }
478 
479     return MOS_STATUS_SUCCESS;
480 }
481 
AllocateSpace(AcquireParams & params,std::vector<MemoryBlock> & blocks)482 MOS_STATUS MemoryBlockManager::AllocateSpace(
483     AcquireParams &params,
484     std::vector<MemoryBlock> &blocks)
485 {
486     HEAP_FUNCTION_ENTER_VERBOSE;
487 
488     if (m_sortedSizes.empty())
489     {
490         HEAP_ASSERTMESSAGE("No space is being requested");
491         return MOS_STATUS_INVALID_PARAMETER;
492     }
493 
494     if (m_sortedBlockList[MemoryBlockInternal::State::free] == nullptr)
495     {
496         HEAP_ASSERTMESSAGE("No free blocks available");
497         return MOS_STATUS_INVALID_PARAMETER;
498     }
499 
500     if (blocks.size() != m_sortedSizes.size())
501     {
502         blocks.resize(m_sortedSizes.size());
503     }
504 
505     for (auto requestIterator = m_sortedSizes.begin();
506         requestIterator != m_sortedSizes.end();
507         ++requestIterator)
508     {
509         bool allocated = false;
510         auto block = m_sortedBlockList[MemoryBlockInternal::State::free];
511         while (block != nullptr)
512         {
513             if (block->GetSize() >= (*requestIterator).m_blockSize)
514             {
515                 auto heap = block->GetHeap();
516                 HEAP_CHK_NULL(heap);
517                 if (!m_useProducer)
518                 {
519                     HEAP_CHK_STATUS(AllocateBlock(
520                         (*requestIterator).m_blockSize,
521                         params.m_trackerId,
522                         params.m_staticBlock,
523                         block));
524                 }
525                 else
526                 {
527                     HEAP_CHK_STATUS(AllocateBlock(
528                         (*requestIterator).m_blockSize,
529                         params.m_trackerIndex,
530                         params.m_trackerId,
531                         params.m_staticBlock,
532                         block));
533                 }
534                 if ((*requestIterator).m_originalIdx >= m_sortedSizes.size())
535                 {
536                     HEAP_ASSERTMESSAGE("Index is out of bounds");
537                     return MOS_STATUS_INVALID_PARAMETER;
538                 }
539                 HEAP_CHK_STATUS(blocks[(*requestIterator).m_originalIdx].CreateFromInternalBlock(
540                     block,
541                     heap,
542                     heap->m_keepLocked ? heap->m_lockedHeap : nullptr));
543                 allocated = true;
544                 break;
545             }
546             block = block->m_stateNext;
547         }
548 
549         if (!allocated)
550         {
551             HEAP_ASSERTMESSAGE("No free block was found for the data! This should not occur.");
552             return MOS_STATUS_UNKNOWN;
553         }
554     }
555 
556     return MOS_STATUS_SUCCESS;
557 }
558 
AllocateBlock(uint32_t alignedSize,uint32_t trackerId,bool staticBlock,MemoryBlockInternal * freeBlock)559 MOS_STATUS MemoryBlockManager::AllocateBlock(
560     uint32_t alignedSize,
561     uint32_t trackerId,
562     bool staticBlock,
563     MemoryBlockInternal *freeBlock)
564 {
565     HEAP_FUNCTION_ENTER_VERBOSE;
566 
567     HEAP_CHK_NULL(freeBlock);
568 
569     if (alignedSize == 0 || alignedSize > freeBlock->GetSize())
570     {
571         HEAP_ASSERTMESSAGE("Size requested is invalid");
572         return MOS_STATUS_INVALID_PARAMETER;
573     }
574 
575     if (freeBlock->GetState() != MemoryBlockInternal::State::free)
576     {
577         HEAP_ASSERTMESSAGE("Free block is not free");
578         return MOS_STATUS_INVALID_PARAMETER;
579     }
580 
581     HEAP_CHK_STATUS(RemoveBlockFromSortedList(freeBlock, freeBlock->GetState()));
582 
583     if (alignedSize < freeBlock->GetSize())
584     {
585         auto remainderBlock = GetBlockFromPool();
586         HEAP_CHK_NULL(remainderBlock);
587         freeBlock->Split(remainderBlock, alignedSize);
588         HEAP_CHK_STATUS(AddBlockToSortedList(remainderBlock, remainderBlock->GetState()));
589     }
590     if (staticBlock)
591     {
592         freeBlock->SetStatic();
593     }
594     HEAP_CHK_STATUS(freeBlock->Allocate(trackerId));
595     HEAP_CHK_STATUS(AddBlockToSortedList(freeBlock, freeBlock->GetState()));
596 
597     return MOS_STATUS_SUCCESS;
598 }
599 
AllocateBlock(uint32_t alignedSize,uint32_t index,uint32_t trackerId,bool staticBlock,MemoryBlockInternal * freeBlock)600 MOS_STATUS MemoryBlockManager::AllocateBlock(
601     uint32_t alignedSize,
602     uint32_t index,
603     uint32_t trackerId,
604     bool staticBlock,
605     MemoryBlockInternal *freeBlock)
606 {
607     HEAP_FUNCTION_ENTER_VERBOSE;
608 
609     HEAP_CHK_NULL(freeBlock);
610 
611     if (!m_useProducer)
612     {
613         HEAP_ASSERTMESSAGE("FrameTrackerProducer need to be registered");
614         return MOS_STATUS_INVALID_PARAMETER;
615     }
616 
617     if (alignedSize == 0 || alignedSize > freeBlock->GetSize())
618     {
619         HEAP_ASSERTMESSAGE("Size requested is invalid");
620         return MOS_STATUS_INVALID_PARAMETER;
621     }
622 
623     if (freeBlock->GetState() != MemoryBlockInternal::State::free)
624     {
625         HEAP_ASSERTMESSAGE("Free block is not free");
626         return MOS_STATUS_INVALID_PARAMETER;
627     }
628 
629     HEAP_CHK_STATUS(RemoveBlockFromSortedList(freeBlock, freeBlock->GetState()));
630 
631     if (alignedSize < freeBlock->GetSize())
632     {
633         auto remainderBlock = GetBlockFromPool();
634         HEAP_CHK_NULL(remainderBlock);
635         freeBlock->Split(remainderBlock, alignedSize);
636         HEAP_CHK_STATUS(AddBlockToSortedList(remainderBlock, remainderBlock->GetState()));
637     }
638     if (staticBlock)
639     {
640         freeBlock->SetStatic();
641     }
642     HEAP_CHK_STATUS(freeBlock->Allocate(index, trackerId, m_trackerProducer));
643     HEAP_CHK_STATUS(AddBlockToSortedList(freeBlock, freeBlock->GetState()));
644 
645     return MOS_STATUS_SUCCESS;
646 }
647 
648 
AddBlockToSortedList(MemoryBlockInternal * block,MemoryBlockInternal::State state)649 MOS_STATUS MemoryBlockManager::AddBlockToSortedList(
650     MemoryBlockInternal *block,
651     MemoryBlockInternal::State state)
652 {
653     HEAP_FUNCTION_ENTER_VERBOSE;
654 
655     HEAP_CHK_NULL(block);
656 
657     if (block->m_statePrev || block->m_stateNext)
658     {
659         HEAP_ASSERTMESSAGE("Inserted blocks should not be in a state list already");
660         return MOS_STATUS_INVALID_PARAMETER;
661     }
662 
663     if (state != block->GetState() ||
664         block->m_stateListType != MemoryBlockInternal::State::stateCount)
665     {
666         HEAP_ASSERTMESSAGE("State does not match list to be added to");
667         return MOS_STATUS_INVALID_PARAMETER;
668     }
669 
670     auto curr = m_sortedBlockList[state];
671 
672     switch (state)
673     {
674         case MemoryBlockInternal::State::free:
675         {
676             bool inserted = false;
677             MemoryBlockInternal *prev = nullptr;
678             while (curr != nullptr)
679             {
680                 if (curr->GetSize() <= block->GetSize())
681                 {
682                     if (prev)
683                     {
684                         prev->m_stateNext = block;
685                     }
686                     else
687                     {
688                         m_sortedBlockList[state] = block;
689                     }
690                     curr->m_statePrev = block;
691                     block->m_statePrev = prev;
692                     block->m_stateNext = curr;
693                     inserted = true;
694                     break;
695                 }
696                 prev = curr;
697                 curr = curr->m_stateNext;
698             }
699             if (!inserted)
700             {
701                 if (prev == nullptr)
702                 {
703                     block->m_stateNext = m_sortedBlockList[state];
704                     m_sortedBlockList[state] = block;
705                 }
706                 else
707                 {
708                     block->m_statePrev = prev;
709                     prev->m_stateNext = block;
710                 }
711             }
712             block->m_stateListType = state;
713             m_sortedBlockListNumEntries[state]++;
714             m_sortedBlockListSizes[state] += block->GetSize();
715             break;
716         }
717         case MemoryBlockInternal::State::allocated:
718         case MemoryBlockInternal::State::submitted:
719         case MemoryBlockInternal::State::deleted:
720             block->m_stateNext = curr;
721             if (curr)
722             {
723                 curr->m_statePrev = block;
724             }
725             m_sortedBlockList[state] = block;
726             block->m_stateListType = state;
727             m_sortedBlockListNumEntries[state]++;
728             m_sortedBlockListSizes[state] += block->GetSize();
729             break;
730         case MemoryBlockInternal::State::pool:
731             block->m_stateNext = curr;
732             if (curr)
733             {
734                 curr->m_statePrev = block;
735             }
736             block->m_stateListType = state;
737             m_sortedBlockList[state] = block;
738             m_sortedBlockListNumEntries[state]++;
739             break;
740         default:
741             HEAP_ASSERTMESSAGE("This state type is unsupported");
742             return MOS_STATUS_INVALID_PARAMETER;
743     }
744 
745     return MOS_STATUS_SUCCESS;
746 }
747 
RemoveBlockFromSortedList(MemoryBlockInternal * block,MemoryBlockInternal::State state)748 MOS_STATUS MemoryBlockManager::RemoveBlockFromSortedList(
749     MemoryBlockInternal *block,
750     MemoryBlockInternal::State state)
751 {
752     HEAP_FUNCTION_ENTER_VERBOSE;
753 
754     HEAP_CHK_NULL(block);
755 
756     switch (state)
757     {
758         case MemoryBlockInternal::State::free:
759         case MemoryBlockInternal::State::allocated:
760         case MemoryBlockInternal::State::submitted:
761         case MemoryBlockInternal::State::deleted:
762         {
763             if (block->m_statePrev)
764             {
765                 block->m_statePrev->m_stateNext = block->m_stateNext;
766             }
767             else
768             {
769                 // special case for beginning of list
770                 m_sortedBlockList[state] = block->m_stateNext;
771             }
772             if (block->m_stateNext)
773             {
774                 block->m_stateNext->m_statePrev = block->m_statePrev;
775             }
776             block->m_statePrev = block->m_stateNext = nullptr;
777             block->m_stateListType = MemoryBlockInternal::State::stateCount;
778             m_sortedBlockListNumEntries[state]--;
779             m_sortedBlockListSizes[state] -= block->GetSize();
780             break;
781         }
782         default:
783             HEAP_ASSERTMESSAGE("This state type is unsupported");
784             return MOS_STATUS_INVALID_PARAMETER;
785             break;
786     }
787 
788     return MOS_STATUS_SUCCESS;
789 }
790 
GetBlockFromPool()791 MemoryBlockInternal *MemoryBlockManager::GetBlockFromPool()
792 {
793     HEAP_FUNCTION_ENTER_VERBOSE;
794 
795     MemoryBlockInternal *block = nullptr;
796 
797     if (m_sortedBlockList[MemoryBlockInternal::State::pool] == nullptr)
798     {
799         block = MOS_New(MemoryBlockInternal);
800     }
801     else
802     {
803         block = m_sortedBlockList[MemoryBlockInternal::State::pool];
804         if (block->m_stateNext)
805         {
806             block->m_stateNext->m_statePrev = nullptr;
807         }
808         // refresh beginning of list
809         m_sortedBlockList[MemoryBlockInternal::State::pool] = block->m_stateNext;
810         block->m_statePrev = block->m_stateNext = nullptr;
811         block->m_stateListType = MemoryBlockInternal::State::stateCount;
812         m_sortedBlockListNumEntries[MemoryBlockInternal::State::pool]--;
813     }
814 
815     return block;
816 }
817 
RemoveHeapFromSortedBlockList(uint32_t heapId)818 MOS_STATUS MemoryBlockManager::RemoveHeapFromSortedBlockList(uint32_t heapId)
819 {
820     for (auto state = 0; state < MemoryBlockInternal::State::stateCount; ++state)
821     {
822         if (state == MemoryBlockInternal::State::pool)
823         {
824             continue;
825         }
826 
827         auto curr = m_sortedBlockList[state];
828         Heap *heap = nullptr;
829         MemoryBlockInternal *nextBlock = nullptr;
830         while (curr != nullptr)
831         {
832             nextBlock = curr->m_stateNext;
833             heap = curr->GetHeap();
834             HEAP_CHK_NULL(heap);
835             if (heap->GetId() == heapId)
836             {
837                 HEAP_CHK_STATUS(RemoveBlockFromSortedList(curr, curr->GetState()));
838             }
839             curr = nextBlock;
840         }
841     }
842 
843     return MOS_STATUS_SUCCESS;
844 }
845 
MergeBlocks(MemoryBlockInternal * blockCombined,MemoryBlockInternal * blockRelease)846 MOS_STATUS MemoryBlockManager::MergeBlocks(
847     MemoryBlockInternal *blockCombined,
848     MemoryBlockInternal *blockRelease)
849 {
850     HEAP_FUNCTION_ENTER_VERBOSE;
851 
852     HEAP_CHK_NULL(blockCombined);
853     HEAP_CHK_NULL(blockRelease);
854 
855     if (blockCombined->GetState() != MemoryBlockInternal::State::free ||
856         blockRelease->GetState() != MemoryBlockInternal::State::free)
857     {
858         HEAP_ASSERTMESSAGE("Only free blocks may be merged");
859         return MOS_STATUS_INVALID_PARAMETER;
860     }
861 
862     HEAP_CHK_STATUS(RemoveBlockFromSortedList(blockCombined, blockCombined->GetState()));
863     HEAP_CHK_STATUS(RemoveBlockFromSortedList(blockRelease, blockRelease->GetState()));
864     HEAP_CHK_STATUS(blockCombined->Combine(blockRelease));
865     HEAP_CHK_STATUS(AddBlockToSortedList(blockRelease, blockRelease->GetState()));
866     HEAP_CHK_STATUS(AddBlockToSortedList(blockCombined, blockCombined->GetState()));
867 
868     return MOS_STATUS_SUCCESS;
869 }
870 
871