1 #define JEMALLOC_TCACHE_C_
2 #include "jemalloc/internal/jemalloc_preamble.h"
3 #include "jemalloc/internal/jemalloc_internal_includes.h"
4
5 #include "jemalloc/internal/assert.h"
6 #include "jemalloc/internal/mutex.h"
7 #include "jemalloc/internal/size_classes.h"
8
9 /******************************************************************************/
10 /* Data. */
11
12 #if !defined(__BIONIC__) || defined(ANDROID_ENABLE_TCACHE)
13 bool opt_tcache = true;
14 #else
15 bool opt_tcache = false;
16 #endif
17 ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
18
19 cache_bin_info_t *tcache_bin_info;
20 static unsigned stack_nelms; /* Total stack elms per tcache. */
21
22 unsigned nhbins;
23 size_t tcache_maxclass;
24
25 tcaches_t *tcaches;
26
27 /* Index of first element within tcaches that has never been used. */
28 static unsigned tcaches_past;
29
30 /* Head of singly linked list tracking available tcaches elements. */
31 static tcaches_t *tcaches_avail;
32
33 /* Protects tcaches{,_past,_avail}. */
34 static malloc_mutex_t tcaches_mtx;
35
36 /******************************************************************************/
37
38 size_t
tcache_salloc(tsdn_t * tsdn,const void * ptr)39 tcache_salloc(tsdn_t *tsdn, const void *ptr) {
40 return arena_salloc(tsdn, ptr);
41 }
42
43 void
tcache_event_hard(tsd_t * tsd,tcache_t * tcache)44 tcache_event_hard(tsd_t *tsd, tcache_t *tcache) {
45 szind_t binind = tcache->next_gc_bin;
46
47 cache_bin_t *tbin;
48 if (binind < NBINS) {
49 tbin = tcache_small_bin_get(tcache, binind);
50 } else {
51 tbin = tcache_large_bin_get(tcache, binind);
52 }
53 if (tbin->low_water > 0) {
54 /*
55 * Flush (ceiling) 3/4 of the objects below the low water mark.
56 */
57 if (binind < NBINS) {
58 tcache_bin_flush_small(tsd, tcache, tbin, binind,
59 tbin->ncached - tbin->low_water + (tbin->low_water
60 >> 2));
61 /*
62 * Reduce fill count by 2X. Limit lg_fill_div such that
63 * the fill count is always at least 1.
64 */
65 cache_bin_info_t *tbin_info = &tcache_bin_info[binind];
66 if ((tbin_info->ncached_max >>
67 (tcache->lg_fill_div[binind] + 1)) >= 1) {
68 tcache->lg_fill_div[binind]++;
69 }
70 } else {
71 tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached
72 - tbin->low_water + (tbin->low_water >> 2), tcache);
73 }
74 } else if (tbin->low_water < 0) {
75 /*
76 * Increase fill count by 2X for small bins. Make sure
77 * lg_fill_div stays greater than 0.
78 */
79 if (binind < NBINS && tcache->lg_fill_div[binind] > 1) {
80 tcache->lg_fill_div[binind]--;
81 }
82 }
83 tbin->low_water = tbin->ncached;
84
85 tcache->next_gc_bin++;
86 if (tcache->next_gc_bin == nhbins) {
87 tcache->next_gc_bin = 0;
88 }
89 }
90
91 void *
tcache_alloc_small_hard(tsdn_t * tsdn,arena_t * arena,tcache_t * tcache,cache_bin_t * tbin,szind_t binind,bool * tcache_success)92 tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache,
93 cache_bin_t *tbin, szind_t binind, bool *tcache_success) {
94 void *ret;
95
96 assert(tcache->arena != NULL);
97 arena_tcache_fill_small(tsdn, arena, tcache, tbin, binind,
98 config_prof ? tcache->prof_accumbytes : 0);
99 if (config_prof) {
100 tcache->prof_accumbytes = 0;
101 }
102 ret = cache_bin_alloc_easy(tbin, tcache_success);
103
104 return ret;
105 }
106
107 void
tcache_bin_flush_small(tsd_t * tsd,tcache_t * tcache,cache_bin_t * tbin,szind_t binind,unsigned rem)108 tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin,
109 szind_t binind, unsigned rem) {
110 bool merged_stats = false;
111
112 assert(binind < NBINS);
113 assert((cache_bin_sz_t)rem <= tbin->ncached);
114
115 arena_t *arena = tcache->arena;
116 assert(arena != NULL);
117 unsigned nflush = tbin->ncached - rem;
118 VARIABLE_ARRAY(extent_t *, item_extent, nflush);
119 /* Look up extent once per item. */
120 for (unsigned i = 0 ; i < nflush; i++) {
121 item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i));
122 }
123
124 while (nflush > 0) {
125 /* Lock the arena bin associated with the first object. */
126 extent_t *extent = item_extent[0];
127 arena_t *bin_arena = extent_arena_get(extent);
128 bin_t *bin = &bin_arena->bins[binind];
129
130 if (config_prof && bin_arena == arena) {
131 if (arena_prof_accum(tsd_tsdn(tsd), arena,
132 tcache->prof_accumbytes)) {
133 prof_idump(tsd_tsdn(tsd));
134 }
135 tcache->prof_accumbytes = 0;
136 }
137
138 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
139 if (config_stats && bin_arena == arena) {
140 assert(!merged_stats);
141 merged_stats = true;
142 #if !defined(ANDROID_MINIMIZE_STRUCTS)
143 bin->stats.nflushes++;
144 #endif
145 #if defined(ANDROID_ENABLE_TCACHE_STATS)
146 bin->stats.nrequests += tbin->tstats.nrequests;
147 tbin->tstats.nrequests = 0;
148 #endif
149 }
150 unsigned ndeferred = 0;
151 for (unsigned i = 0; i < nflush; i++) {
152 void *ptr = *(tbin->avail - 1 - i);
153 extent = item_extent[i];
154 assert(ptr != NULL && extent != NULL);
155
156 if (extent_arena_get(extent) == bin_arena) {
157 arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
158 bin_arena, extent, ptr);
159 } else {
160 /*
161 * This object was allocated via a different
162 * arena bin than the one that is currently
163 * locked. Stash the object, so that it can be
164 * handled in a future pass.
165 */
166 *(tbin->avail - 1 - ndeferred) = ptr;
167 item_extent[ndeferred] = extent;
168 ndeferred++;
169 }
170 }
171 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
172 arena_decay_ticks(tsd_tsdn(tsd), bin_arena, nflush - ndeferred);
173 nflush = ndeferred;
174 }
175 if (config_stats && !merged_stats) {
176 /*
177 * The flush loop didn't happen to flush to this thread's
178 * arena, so the stats didn't get merged. Manually do so now.
179 */
180 bin_t *bin = &arena->bins[binind];
181 malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
182 #if !defined(ANDROID_MINIMIZE_STRUCTS)
183 bin->stats.nflushes++;
184 #endif
185 #if defined(ANDROID_ENABLE_TCACHE_STATS)
186 bin->stats.nrequests += tbin->tstats.nrequests;
187 tbin->tstats.nrequests = 0;
188 #endif
189 malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock);
190 }
191
192 memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
193 sizeof(void *));
194 tbin->ncached = rem;
195 if (tbin->ncached < tbin->low_water) {
196 tbin->low_water = tbin->ncached;
197 }
198 }
199
200 void
tcache_bin_flush_large(tsd_t * tsd,cache_bin_t * tbin,szind_t binind,unsigned rem,tcache_t * tcache)201 tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind,
202 unsigned rem, tcache_t *tcache) {
203 #if defined(ANDROID_ENABLE_TCACHE_STATS)
204 bool merged_stats = false;
205 #endif
206
207 assert(binind < nhbins);
208 assert((cache_bin_sz_t)rem <= tbin->ncached);
209
210 arena_t *arena = tcache->arena;
211 assert(arena != NULL);
212 unsigned nflush = tbin->ncached - rem;
213 VARIABLE_ARRAY(extent_t *, item_extent, nflush);
214 /* Look up extent once per item. */
215 for (unsigned i = 0 ; i < nflush; i++) {
216 item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i));
217 }
218
219 while (nflush > 0) {
220 /* Lock the arena associated with the first object. */
221 extent_t *extent = item_extent[0];
222 arena_t *locked_arena = extent_arena_get(extent);
223 UNUSED bool idump;
224
225 if (config_prof) {
226 idump = false;
227 }
228
229 malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx);
230 for (unsigned i = 0; i < nflush; i++) {
231 void *ptr = *(tbin->avail - 1 - i);
232 assert(ptr != NULL);
233 extent = item_extent[i];
234 if (extent_arena_get(extent) == locked_arena) {
235 large_dalloc_prep_junked_locked(tsd_tsdn(tsd),
236 extent);
237 }
238 }
239 if ((config_prof || config_stats) && locked_arena == arena) {
240 if (config_prof) {
241 idump = arena_prof_accum(tsd_tsdn(tsd), arena,
242 tcache->prof_accumbytes);
243 tcache->prof_accumbytes = 0;
244 }
245 #if defined(ANDROID_ENABLE_TCACHE_STATS)
246 if (config_stats) {
247 merged_stats = true;
248 arena_stats_large_nrequests_add(tsd_tsdn(tsd),
249 &arena->stats, binind,
250 tbin->tstats.nrequests);
251 tbin->tstats.nrequests = 0;
252 }
253 #endif
254 }
255 malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx);
256
257 unsigned ndeferred = 0;
258 for (unsigned i = 0; i < nflush; i++) {
259 void *ptr = *(tbin->avail - 1 - i);
260 extent = item_extent[i];
261 assert(ptr != NULL && extent != NULL);
262
263 if (extent_arena_get(extent) == locked_arena) {
264 large_dalloc_finish(tsd_tsdn(tsd), extent);
265 } else {
266 /*
267 * This object was allocated via a different
268 * arena than the one that is currently locked.
269 * Stash the object, so that it can be handled
270 * in a future pass.
271 */
272 *(tbin->avail - 1 - ndeferred) = ptr;
273 item_extent[ndeferred] = extent;
274 ndeferred++;
275 }
276 }
277 if (config_prof && idump) {
278 prof_idump(tsd_tsdn(tsd));
279 }
280 arena_decay_ticks(tsd_tsdn(tsd), locked_arena, nflush -
281 ndeferred);
282 nflush = ndeferred;
283 }
284 #if defined(ANDROID_ENABLE_TCACHE_STATS)
285 if (config_stats && !merged_stats) {
286 /*
287 * The flush loop didn't happen to flush to this thread's
288 * arena, so the stats didn't get merged. Manually do so now.
289 */
290 arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats,
291 binind, tbin->tstats.nrequests);
292 tbin->tstats.nrequests = 0;
293 }
294 #endif
295
296 memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem *
297 sizeof(void *));
298 tbin->ncached = rem;
299 if (tbin->ncached < tbin->low_water) {
300 tbin->low_water = tbin->ncached;
301 }
302 }
303
304 void
tcache_arena_associate(tsdn_t * tsdn,tcache_t * tcache,arena_t * arena)305 tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
306 #if defined(ANDROID_ENABLE_TCACHE)
307 assert(tcache->arena == NULL);
308 tcache->arena = arena;
309
310 if (config_stats) {
311 /* Link into list of extant tcaches. */
312 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
313
314 ql_elm_new(tcache, link);
315 ql_tail_insert(&arena->tcache_ql, tcache, link);
316 cache_bin_array_descriptor_init(
317 &tcache->cache_bin_array_descriptor, tcache->bins_small,
318 tcache->bins_large);
319 ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
320 &tcache->cache_bin_array_descriptor, link);
321
322 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
323 }
324 #endif
325 }
326
327 static void
tcache_arena_dissociate(tsdn_t * tsdn,tcache_t * tcache)328 tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) {
329 #if defined(ANDROID_ENABLE_TCACHE)
330 arena_t *arena = tcache->arena;
331 assert(arena != NULL);
332 if (config_stats) {
333 /* Unlink from list of extant tcaches. */
334 malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
335 if (config_debug) {
336 bool in_ql = false;
337 tcache_t *iter;
338 ql_foreach(iter, &arena->tcache_ql, link) {
339 if (iter == tcache) {
340 in_ql = true;
341 break;
342 }
343 }
344 assert(in_ql);
345 }
346 ql_remove(&arena->tcache_ql, tcache, link);
347 ql_remove(&arena->cache_bin_array_descriptor_ql,
348 &tcache->cache_bin_array_descriptor, link);
349 tcache_stats_merge(tsdn, tcache, arena);
350 malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
351 }
352 tcache->arena = NULL;
353 #endif
354 }
355
356 void
tcache_arena_reassociate(tsdn_t * tsdn,tcache_t * tcache,arena_t * arena)357 tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
358 tcache_arena_dissociate(tsdn, tcache);
359 tcache_arena_associate(tsdn, tcache, arena);
360 }
361
362 bool
tsd_tcache_enabled_data_init(tsd_t * tsd)363 tsd_tcache_enabled_data_init(tsd_t *tsd) {
364 /* Called upon tsd initialization. */
365 tsd_tcache_enabled_set(tsd, opt_tcache);
366 tsd_slow_update(tsd);
367
368 if (opt_tcache) {
369 /* Trigger tcache init. */
370 tsd_tcache_data_init(tsd);
371 }
372
373 return false;
374 }
375
376 /* Initialize auto tcache (embedded in TSD). */
377 static void
tcache_init(tsd_t * tsd,tcache_t * tcache,void * avail_stack)378 tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) {
379 memset(&tcache->link, 0, sizeof(ql_elm(tcache_t)));
380 tcache->prof_accumbytes = 0;
381 tcache->next_gc_bin = 0;
382 tcache->arena = NULL;
383
384 ticker_init(&tcache->gc_ticker, TCACHE_GC_INCR);
385
386 size_t stack_offset = 0;
387 assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
388 memset(tcache->bins_small, 0, sizeof(cache_bin_t) * NBINS);
389 memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - NBINS));
390 unsigned i = 0;
391 for (; i < NBINS; i++) {
392 tcache->lg_fill_div[i] = 1;
393 stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
394 /*
395 * avail points past the available space. Allocations will
396 * access the slots toward higher addresses (for the benefit of
397 * prefetch).
398 */
399 tcache_small_bin_get(tcache, i)->avail =
400 (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset);
401 }
402 for (; i < nhbins; i++) {
403 stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
404 tcache_large_bin_get(tcache, i)->avail =
405 (void **)((uintptr_t)avail_stack + (uintptr_t)stack_offset);
406 }
407 assert(stack_offset == stack_nelms * sizeof(void *));
408 }
409
410 /* Initialize auto tcache (embedded in TSD). */
411 bool
tsd_tcache_data_init(tsd_t * tsd)412 tsd_tcache_data_init(tsd_t *tsd) {
413 tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
414 assert(tcache_small_bin_get(tcache, 0)->avail == NULL);
415 size_t size = stack_nelms * sizeof(void *);
416 /* Avoid false cacheline sharing. */
417 size = sz_sa2u(size, CACHELINE);
418
419 void *avail_array = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true,
420 NULL, true, arena_get(TSDN_NULL, 0, true));
421 if (avail_array == NULL) {
422 return true;
423 }
424
425 tcache_init(tsd, tcache, avail_array);
426 /*
427 * Initialization is a bit tricky here. After malloc init is done, all
428 * threads can rely on arena_choose and associate tcache accordingly.
429 * However, the thread that does actual malloc bootstrapping relies on
430 * functional tsd, and it can only rely on a0. In that case, we
431 * associate its tcache to a0 temporarily, and later on
432 * arena_choose_hard() will re-associate properly.
433 */
434 tcache->arena = NULL;
435 arena_t *arena;
436 if (!malloc_initialized()) {
437 /* If in initialization, assign to a0. */
438 arena = arena_get(tsd_tsdn(tsd), 0, false);
439 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
440 } else {
441 arena = arena_choose(tsd, NULL);
442 /* This may happen if thread.tcache.enabled is used. */
443 if (tcache->arena == NULL) {
444 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena);
445 }
446 }
447 assert(arena == tcache->arena);
448
449 return false;
450 }
451
452 /* Created manual tcache for tcache.create mallctl. */
453 tcache_t *
tcache_create_explicit(tsd_t * tsd)454 tcache_create_explicit(tsd_t *tsd) {
455 tcache_t *tcache;
456 size_t size, stack_offset;
457
458 size = sizeof(tcache_t);
459 /* Naturally align the pointer stacks. */
460 size = PTR_CEILING(size);
461 stack_offset = size;
462 size += stack_nelms * sizeof(void *);
463 /* Avoid false cacheline sharing. */
464 size = sz_sa2u(size, CACHELINE);
465
466 tcache = ipallocztm(tsd_tsdn(tsd), size, CACHELINE, true, NULL, true,
467 arena_get(TSDN_NULL, 0, true));
468 if (tcache == NULL) {
469 return NULL;
470 }
471
472 tcache_init(tsd, tcache,
473 (void *)((uintptr_t)tcache + (uintptr_t)stack_offset));
474 tcache_arena_associate(tsd_tsdn(tsd), tcache, arena_ichoose(tsd, NULL));
475
476 return tcache;
477 }
478
479 static void
tcache_flush_cache(tsd_t * tsd,tcache_t * tcache)480 tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
481 assert(tcache->arena != NULL);
482
483 for (unsigned i = 0; i < NBINS; i++) {
484 cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
485 tcache_bin_flush_small(tsd, tcache, tbin, i, 0);
486
487 #if defined(ANDROID_ENABLE_TCACHE_STATS)
488 if (config_stats) {
489 assert(tbin->tstats.nrequests == 0);
490 }
491 #endif
492 }
493 for (unsigned i = NBINS; i < nhbins; i++) {
494 cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
495 tcache_bin_flush_large(tsd, tbin, i, 0, tcache);
496
497 #if defined(ANDROID_ENABLE_TCACHE_STATS)
498 if (config_stats) {
499 assert(tbin->tstats.nrequests == 0);
500 }
501 #endif
502 }
503
504 if (config_prof && tcache->prof_accumbytes > 0 &&
505 arena_prof_accum(tsd_tsdn(tsd), tcache->arena,
506 tcache->prof_accumbytes)) {
507 prof_idump(tsd_tsdn(tsd));
508 }
509 }
510
511 void
tcache_flush(tsd_t * tsd)512 tcache_flush(tsd_t *tsd) {
513 assert(tcache_available(tsd));
514 tcache_flush_cache(tsd, tsd_tcachep_get(tsd));
515 }
516
517 static void
tcache_destroy(tsd_t * tsd,tcache_t * tcache,bool tsd_tcache)518 tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
519 tcache_flush_cache(tsd, tcache);
520 tcache_arena_dissociate(tsd_tsdn(tsd), tcache);
521
522 if (tsd_tcache) {
523 /* Release the avail array for the TSD embedded auto tcache. */
524 void *avail_array =
525 (void *)((uintptr_t)tcache_small_bin_get(tcache, 0)->avail -
526 (uintptr_t)tcache_bin_info[0].ncached_max * sizeof(void *));
527 idalloctm(tsd_tsdn(tsd), avail_array, NULL, NULL, true, true);
528 } else {
529 /* Release both the tcache struct and avail array. */
530 idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true);
531 }
532 }
533
534 /* For auto tcache (embedded in TSD) only. */
535 void
tcache_cleanup(tsd_t * tsd)536 tcache_cleanup(tsd_t *tsd) {
537 tcache_t *tcache = tsd_tcachep_get(tsd);
538 if (!tcache_available(tsd)) {
539 assert(tsd_tcache_enabled_get(tsd) == false);
540 if (config_debug) {
541 assert(tcache_small_bin_get(tcache, 0)->avail == NULL);
542 }
543 return;
544 }
545 assert(tsd_tcache_enabled_get(tsd));
546 assert(tcache_small_bin_get(tcache, 0)->avail != NULL);
547
548 tcache_destroy(tsd, tcache, true);
549 if (config_debug) {
550 tcache_small_bin_get(tcache, 0)->avail = NULL;
551 }
552 }
553
554 void
tcache_stats_merge(tsdn_t * tsdn,tcache_t * tcache,arena_t * arena)555 tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
556 #if defined(ANDROID_ENABLE_TCACHE_STATS)
557 unsigned i;
558
559 cassert(config_stats);
560
561 /* Merge and reset tcache stats. */
562 for (i = 0; i < NBINS; i++) {
563 bin_t *bin = &arena->bins[i];
564 cache_bin_t *tbin = tcache_small_bin_get(tcache, i);
565 malloc_mutex_lock(tsdn, &bin->lock);
566 bin->stats.nrequests += tbin->tstats.nrequests;
567 malloc_mutex_unlock(tsdn, &bin->lock);
568 tbin->tstats.nrequests = 0;
569 }
570
571 for (; i < nhbins; i++) {
572 cache_bin_t *tbin = tcache_large_bin_get(tcache, i);
573 arena_stats_large_nrequests_add(tsdn, &arena->stats, i,
574 tbin->tstats.nrequests);
575 tbin->tstats.nrequests = 0;
576 }
577 #endif
578 }
579
580 static bool
tcaches_create_prep(tsd_t * tsd)581 tcaches_create_prep(tsd_t *tsd) {
582 bool err;
583
584 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
585
586 if (tcaches == NULL) {
587 tcaches = base_alloc(tsd_tsdn(tsd), b0get(), sizeof(tcache_t *)
588 * (MALLOCX_TCACHE_MAX+1), CACHELINE);
589 if (tcaches == NULL) {
590 err = true;
591 goto label_return;
592 }
593 }
594
595 if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) {
596 err = true;
597 goto label_return;
598 }
599
600 err = false;
601 label_return:
602 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
603 return err;
604 }
605
606 bool
tcaches_create(tsd_t * tsd,unsigned * r_ind)607 tcaches_create(tsd_t *tsd, unsigned *r_ind) {
608 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
609
610 bool err;
611
612 if (tcaches_create_prep(tsd)) {
613 err = true;
614 goto label_return;
615 }
616
617 tcache_t *tcache = tcache_create_explicit(tsd);
618 if (tcache == NULL) {
619 err = true;
620 goto label_return;
621 }
622
623 tcaches_t *elm;
624 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
625 if (tcaches_avail != NULL) {
626 elm = tcaches_avail;
627 tcaches_avail = tcaches_avail->next;
628 elm->tcache = tcache;
629 *r_ind = (unsigned)(elm - tcaches);
630 } else {
631 elm = &tcaches[tcaches_past];
632 elm->tcache = tcache;
633 *r_ind = tcaches_past;
634 tcaches_past++;
635 }
636 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
637
638 err = false;
639 label_return:
640 witness_assert_depth(tsdn_witness_tsdp_get(tsd_tsdn(tsd)), 0);
641 return err;
642 }
643
644 static tcache_t *
tcaches_elm_remove(tsd_t * tsd,tcaches_t * elm)645 tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) {
646 malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx);
647
648 if (elm->tcache == NULL) {
649 return NULL;
650 }
651 tcache_t *tcache = elm->tcache;
652 elm->tcache = NULL;
653 return tcache;
654 }
655
656 void
tcaches_flush(tsd_t * tsd,unsigned ind)657 tcaches_flush(tsd_t *tsd, unsigned ind) {
658 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
659 tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]);
660 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
661 if (tcache != NULL) {
662 tcache_destroy(tsd, tcache, false);
663 }
664 }
665
666 void
tcaches_destroy(tsd_t * tsd,unsigned ind)667 tcaches_destroy(tsd_t *tsd, unsigned ind) {
668 malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx);
669 tcaches_t *elm = &tcaches[ind];
670 tcache_t *tcache = tcaches_elm_remove(tsd, elm);
671 elm->next = tcaches_avail;
672 tcaches_avail = elm;
673 malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx);
674 if (tcache != NULL) {
675 tcache_destroy(tsd, tcache, false);
676 }
677 }
678
679 bool
tcache_boot(tsdn_t * tsdn)680 tcache_boot(tsdn_t *tsdn) {
681 /* If necessary, clamp opt_lg_tcache_max. */
682 if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) <
683 SMALL_MAXCLASS) {
684 tcache_maxclass = SMALL_MAXCLASS;
685 } else {
686 tcache_maxclass = (ZU(1) << opt_lg_tcache_max);
687 }
688
689 if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,
690 malloc_mutex_rank_exclusive)) {
691 return true;
692 }
693
694 nhbins = sz_size2index(tcache_maxclass) + 1;
695
696 /* Initialize tcache_bin_info. */
697 tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins
698 * sizeof(cache_bin_info_t), CACHELINE);
699 if (tcache_bin_info == NULL) {
700 return true;
701 }
702 stack_nelms = 0;
703 unsigned i;
704 for (i = 0; i < NBINS; i++) {
705 if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) {
706 tcache_bin_info[i].ncached_max =
707 TCACHE_NSLOTS_SMALL_MIN;
708 } else if ((bin_infos[i].nregs << 1) <=
709 TCACHE_NSLOTS_SMALL_MAX) {
710 tcache_bin_info[i].ncached_max =
711 (bin_infos[i].nregs << 1);
712 } else {
713 tcache_bin_info[i].ncached_max =
714 TCACHE_NSLOTS_SMALL_MAX;
715 }
716 stack_nelms += tcache_bin_info[i].ncached_max;
717 }
718 for (; i < nhbins; i++) {
719 tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
720 stack_nelms += tcache_bin_info[i].ncached_max;
721 }
722
723 return false;
724 }
725
726 void
tcache_prefork(tsdn_t * tsdn)727 tcache_prefork(tsdn_t *tsdn) {
728 if (!config_prof && opt_tcache) {
729 malloc_mutex_prefork(tsdn, &tcaches_mtx);
730 }
731 }
732
733 void
tcache_postfork_parent(tsdn_t * tsdn)734 tcache_postfork_parent(tsdn_t *tsdn) {
735 if (!config_prof && opt_tcache) {
736 malloc_mutex_postfork_parent(tsdn, &tcaches_mtx);
737 }
738 }
739
740 void
tcache_postfork_child(tsdn_t * tsdn)741 tcache_postfork_child(tsdn_t *tsdn) {
742 if (!config_prof && opt_tcache) {
743 malloc_mutex_postfork_child(tsdn, &tcaches_mtx);
744 }
745 }
746