1 // Copyright 2022 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef BASE_ALLOCATOR_DISPATCHER_INTERNAL_DISPATCHER_INTERNAL_H_
6 #define BASE_ALLOCATOR_DISPATCHER_INTERNAL_DISPATCHER_INTERNAL_H_
7
8 #include "base/allocator/dispatcher/configuration.h"
9 #include "base/allocator/dispatcher/internal/dispatch_data.h"
10 #include "base/allocator/dispatcher/internal/tools.h"
11 #include "base/allocator/dispatcher/memory_tagging.h"
12 #include "base/allocator/dispatcher/notification_data.h"
13 #include "base/allocator/dispatcher/subsystem.h"
14 #include "base/check.h"
15 #include "base/compiler_specific.h"
16 #include "partition_alloc/partition_alloc_buildflags.h"
17
18 #if BUILDFLAG(USE_PARTITION_ALLOC)
19 #include "partition_alloc/partition_alloc_allocation_data.h"
20 #endif
21
22 #if BUILDFLAG(USE_ALLOCATOR_SHIM)
23 #include "partition_alloc/shim/allocator_shim.h"
24 #endif
25
26 #include <tuple>
27
28 namespace base::allocator::dispatcher::internal {
29
30 #if BUILDFLAG(USE_ALLOCATOR_SHIM)
31 using allocator_shim::AllocatorDispatch;
32 #endif
33
34 template <typename CheckObserverPredicate,
35 typename... ObserverTypes,
36 size_t... Indices>
PerformObserverCheck(const std::tuple<ObserverTypes...> & observers,std::index_sequence<Indices...>,CheckObserverPredicate check_observer)37 void inline PerformObserverCheck(const std::tuple<ObserverTypes...>& observers,
38 std::index_sequence<Indices...>,
39 CheckObserverPredicate check_observer) {
40 ([](bool b) { DCHECK(b); }(check_observer(std::get<Indices>(observers))),
41 ...);
42 }
43
44 template <typename... ObserverTypes, size_t... Indices>
PerformAllocationNotification(const std::tuple<ObserverTypes...> & observers,std::index_sequence<Indices...>,const AllocationNotificationData & notification_data)45 ALWAYS_INLINE void PerformAllocationNotification(
46 const std::tuple<ObserverTypes...>& observers,
47 std::index_sequence<Indices...>,
48 const AllocationNotificationData& notification_data) {
49 ((std::get<Indices>(observers)->OnAllocation(notification_data)), ...);
50 }
51
52 template <typename... ObserverTypes, size_t... Indices>
PerformFreeNotification(const std::tuple<ObserverTypes...> & observers,std::index_sequence<Indices...>,const FreeNotificationData & notification_data)53 ALWAYS_INLINE void PerformFreeNotification(
54 const std::tuple<ObserverTypes...>& observers,
55 std::index_sequence<Indices...>,
56 const FreeNotificationData& notification_data) {
57 ((std::get<Indices>(observers)->OnFree(notification_data)), ...);
58 }
59
60 // DispatcherImpl provides hooks into the various memory subsystems. These hooks
61 // are responsible for dispatching any notification to the observers.
62 // In order to provide as many information on the exact type of the observer and
63 // prevent any conditional jumps in the hot allocation path, observers are
64 // stored in a std::tuple. DispatcherImpl performs a CHECK at initialization
65 // time to ensure they are valid.
66 template <typename... ObserverTypes>
67 struct DispatcherImpl {
68 using AllObservers = std::index_sequence_for<ObserverTypes...>;
69
70 template <std::enable_if_t<
71 internal::LessEqual(sizeof...(ObserverTypes),
72 configuration::kMaximumNumberOfObservers),
73 bool> = true>
GetNotificationHooksDispatcherImpl74 static DispatchData GetNotificationHooks(
75 std::tuple<ObserverTypes*...> observers) {
76 s_observers = std::move(observers);
77
78 PerformObserverCheck(s_observers, AllObservers{}, IsValidObserver{});
79
80 return CreateDispatchData();
81 }
82
83 private:
CreateDispatchDataDispatcherImpl84 static DispatchData CreateDispatchData() {
85 return DispatchData()
86 #if BUILDFLAG(USE_PARTITION_ALLOC)
87 .SetAllocationObserverHooks(&PartitionAllocatorAllocationHook,
88 &PartitionAllocatorFreeHook)
89 #endif
90 #if BUILDFLAG(USE_ALLOCATOR_SHIM)
91 .SetAllocatorDispatch(&allocator_dispatch_)
92 #endif
93 ;
94 }
95
96 #if BUILDFLAG(USE_PARTITION_ALLOC)
PartitionAllocatorAllocationHookDispatcherImpl97 static void PartitionAllocatorAllocationHook(
98 const partition_alloc::AllocationNotificationData& pa_notification_data) {
99 AllocationNotificationData dispatcher_notification_data(
100 pa_notification_data.address(), pa_notification_data.size(),
101 pa_notification_data.type_name(),
102 AllocationSubsystem::kPartitionAllocator);
103
104 #if BUILDFLAG(HAS_MEMORY_TAGGING)
105 dispatcher_notification_data.SetMteReportingMode(
106 ConvertToMTEMode(pa_notification_data.mte_reporting_mode()));
107 #endif
108
109 DoNotifyAllocation(dispatcher_notification_data);
110 }
111
PartitionAllocatorFreeHookDispatcherImpl112 static void PartitionAllocatorFreeHook(
113 const partition_alloc::FreeNotificationData& pa_notification_data) {
114 FreeNotificationData dispatcher_notification_data(
115 pa_notification_data.address(),
116 AllocationSubsystem::kPartitionAllocator);
117
118 #if BUILDFLAG(HAS_MEMORY_TAGGING)
119 dispatcher_notification_data.SetMteReportingMode(
120 ConvertToMTEMode(pa_notification_data.mte_reporting_mode()));
121 #endif
122
123 DoNotifyFree(dispatcher_notification_data);
124 }
125 #endif // BUILDFLAG(USE_PARTITION_ALLOC)
126
127 #if BUILDFLAG(USE_ALLOCATOR_SHIM)
AllocFnDispatcherImpl128 static void* AllocFn(const AllocatorDispatch* self,
129 size_t size,
130 void* context) {
131 void* const address = self->next->alloc_function(self->next, size, context);
132
133 DoNotifyAllocationForShim(address, size);
134
135 return address;
136 }
137
AllocUncheckedFnDispatcherImpl138 static void* AllocUncheckedFn(const AllocatorDispatch* self,
139 size_t size,
140 void* context) {
141 void* const address =
142 self->next->alloc_unchecked_function(self->next, size, context);
143
144 DoNotifyAllocationForShim(address, size);
145
146 return address;
147 }
148
AllocZeroInitializedFnDispatcherImpl149 static void* AllocZeroInitializedFn(const AllocatorDispatch* self,
150 size_t n,
151 size_t size,
152 void* context) {
153 void* const address = self->next->alloc_zero_initialized_function(
154 self->next, n, size, context);
155
156 DoNotifyAllocationForShim(address, n * size);
157
158 return address;
159 }
160
AllocAlignedFnDispatcherImpl161 static void* AllocAlignedFn(const AllocatorDispatch* self,
162 size_t alignment,
163 size_t size,
164 void* context) {
165 void* const address = self->next->alloc_aligned_function(
166 self->next, alignment, size, context);
167
168 DoNotifyAllocationForShim(address, size);
169
170 return address;
171 }
172
ReallocFnDispatcherImpl173 static void* ReallocFn(const AllocatorDispatch* self,
174 void* address,
175 size_t size,
176 void* context) {
177 // Note: size == 0 actually performs free.
178 DoNotifyFreeForShim(address);
179 void* const reallocated_address =
180 self->next->realloc_function(self->next, address, size, context);
181
182 DoNotifyAllocationForShim(reallocated_address, size);
183
184 return reallocated_address;
185 }
186
FreeFnDispatcherImpl187 static void FreeFn(const AllocatorDispatch* self,
188 void* address,
189 void* context) {
190 // Note: DoNotifyFree should be called before free_function (here and in
191 // other places). That is because observers need to handle the allocation
192 // being freed before calling free_function, as once the latter is executed
193 // the address becomes available and can be allocated by another thread.
194 // That would be racy otherwise.
195 DoNotifyFreeForShim(address);
196 self->next->free_function(self->next, address, context);
197 }
198
GetSizeEstimateFnDispatcherImpl199 static size_t GetSizeEstimateFn(const AllocatorDispatch* self,
200 void* address,
201 void* context) {
202 return self->next->get_size_estimate_function(self->next, address, context);
203 }
204
GoodSizeFnDispatcherImpl205 static size_t GoodSizeFn(const AllocatorDispatch* self,
206 size_t size,
207 void* context) {
208 return self->next->good_size_function(self->next, size, context);
209 }
210
ClaimedAddressFnDispatcherImpl211 static bool ClaimedAddressFn(const AllocatorDispatch* self,
212 void* address,
213 void* context) {
214 return self->next->claimed_address_function(self->next, address, context);
215 }
216
BatchMallocFnDispatcherImpl217 static unsigned BatchMallocFn(const AllocatorDispatch* self,
218 size_t size,
219 void** results,
220 unsigned num_requested,
221 void* context) {
222 unsigned const num_allocated = self->next->batch_malloc_function(
223 self->next, size, results, num_requested, context);
224 for (unsigned i = 0; i < num_allocated; ++i) {
225 DoNotifyAllocationForShim(results[i], size);
226 }
227 return num_allocated;
228 }
229
BatchFreeFnDispatcherImpl230 static void BatchFreeFn(const AllocatorDispatch* self,
231 void** to_be_freed,
232 unsigned num_to_be_freed,
233 void* context) {
234 for (unsigned i = 0; i < num_to_be_freed; ++i) {
235 DoNotifyFreeForShim(to_be_freed[i]);
236 }
237
238 self->next->batch_free_function(self->next, to_be_freed, num_to_be_freed,
239 context);
240 }
241
FreeDefiniteSizeFnDispatcherImpl242 static void FreeDefiniteSizeFn(const AllocatorDispatch* self,
243 void* address,
244 size_t size,
245 void* context) {
246 DoNotifyFreeForShim(address);
247 self->next->free_definite_size_function(self->next, address, size, context);
248 }
249
TryFreeDefaultFnDispatcherImpl250 static void TryFreeDefaultFn(const AllocatorDispatch* self,
251 void* address,
252 void* context) {
253 DoNotifyFreeForShim(address);
254 self->next->try_free_default_function(self->next, address, context);
255 }
256
AlignedMallocFnDispatcherImpl257 static void* AlignedMallocFn(const AllocatorDispatch* self,
258 size_t size,
259 size_t alignment,
260 void* context) {
261 void* const address = self->next->aligned_malloc_function(
262 self->next, size, alignment, context);
263
264 DoNotifyAllocationForShim(address, size);
265
266 return address;
267 }
268
AlignedReallocFnDispatcherImpl269 static void* AlignedReallocFn(const AllocatorDispatch* self,
270 void* address,
271 size_t size,
272 size_t alignment,
273 void* context) {
274 // Note: size == 0 actually performs free.
275 DoNotifyFreeForShim(address);
276 address = self->next->aligned_realloc_function(self->next, address, size,
277 alignment, context);
278
279 DoNotifyAllocationForShim(address, size);
280
281 return address;
282 }
283
AlignedFreeFnDispatcherImpl284 static void AlignedFreeFn(const AllocatorDispatch* self,
285 void* address,
286 void* context) {
287 DoNotifyFreeForShim(address);
288 self->next->aligned_free_function(self->next, address, context);
289 }
290
DoNotifyAllocationForShimDispatcherImpl291 ALWAYS_INLINE static void DoNotifyAllocationForShim(void* address,
292 size_t size) {
293 AllocationNotificationData notification_data(
294 address, size, nullptr, AllocationSubsystem::kAllocatorShim);
295
296 DoNotifyAllocation(notification_data);
297 }
298
DoNotifyFreeForShimDispatcherImpl299 ALWAYS_INLINE static void DoNotifyFreeForShim(void* address) {
300 FreeNotificationData notification_data(address,
301 AllocationSubsystem::kAllocatorShim);
302
303 DoNotifyFree(notification_data);
304 }
305
306 static AllocatorDispatch allocator_dispatch_;
307 #endif // BUILDFLAG(USE_ALLOCATOR_SHIM)
308
DoNotifyAllocationDispatcherImpl309 ALWAYS_INLINE static void DoNotifyAllocation(
310 const AllocationNotificationData& notification_data) {
311 PerformAllocationNotification(s_observers, AllObservers{},
312 notification_data);
313 }
314
DoNotifyFreeDispatcherImpl315 ALWAYS_INLINE static void DoNotifyFree(
316 const FreeNotificationData& notification_data) {
317 PerformFreeNotification(s_observers, AllObservers{}, notification_data);
318 }
319
320 static std::tuple<ObserverTypes*...> s_observers;
321 };
322
323 template <typename... ObserverTypes>
324 std::tuple<ObserverTypes*...> DispatcherImpl<ObserverTypes...>::s_observers;
325
326 #if BUILDFLAG(USE_ALLOCATOR_SHIM)
327 template <typename... ObserverTypes>
328 AllocatorDispatch DispatcherImpl<ObserverTypes...>::allocator_dispatch_ = {
329 &AllocFn,
330 &AllocUncheckedFn,
331 &AllocZeroInitializedFn,
332 &AllocAlignedFn,
333 &ReallocFn,
334 &FreeFn,
335 &GetSizeEstimateFn,
336 &GoodSizeFn,
337 &ClaimedAddressFn,
338 &BatchMallocFn,
339 &BatchFreeFn,
340 &FreeDefiniteSizeFn,
341 &TryFreeDefaultFn,
342 &AlignedMallocFn,
343 &AlignedReallocFn,
344 &AlignedFreeFn,
345 nullptr};
346 #endif // BUILDFLAG(USE_ALLOCATOR_SHIM)
347
348 // Specialization of DispatcherImpl in case we have no observers to notify. In
349 // this special case we return a set of null pointers as the Dispatcher must not
350 // install any hooks at all.
351 template <>
352 struct DispatcherImpl<> {
353 static DispatchData GetNotificationHooks(std::tuple<> /*observers*/) {
354 return DispatchData()
355 #if BUILDFLAG(USE_PARTITION_ALLOC)
356 .SetAllocationObserverHooks(nullptr, nullptr)
357 #endif
358 #if BUILDFLAG(USE_ALLOCATOR_SHIM)
359 .SetAllocatorDispatch(nullptr)
360 #endif
361 ;
362 }
363 };
364
365 // A little utility function that helps using DispatcherImpl by providing
366 // automated type deduction for templates.
367 template <typename... ObserverTypes>
368 inline DispatchData GetNotificationHooks(
369 std::tuple<ObserverTypes*...> observers) {
370 return DispatcherImpl<ObserverTypes...>::GetNotificationHooks(
371 std::move(observers));
372 }
373
374 } // namespace base::allocator::dispatcher::internal
375
376 #endif // BASE_ALLOCATOR_DISPATCHER_INTERNAL_DISPATCHER_INTERNAL_H_
377