1 // Copyright 2017 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include <stdint.h>
16
17 #include <new>
18
19 // This file is a no-op if the required LowLevelAlloc support is missing.
20 #include "absl/base/internal/low_level_alloc.h"
21 #include "absl/synchronization/internal/waiter.h"
22 #ifndef ABSL_LOW_LEVEL_ALLOC_MISSING
23
24 #include <string.h>
25
26 #include "absl/base/attributes.h"
27 #include "absl/base/internal/spinlock.h"
28 #include "absl/base/internal/thread_identity.h"
29 #include "absl/synchronization/internal/per_thread_sem.h"
30
31 namespace absl {
32 ABSL_NAMESPACE_BEGIN
33 namespace synchronization_internal {
34
35 // ThreadIdentity storage is persistent, we maintain a free-list of previously
36 // released ThreadIdentity objects.
37 ABSL_CONST_INIT static base_internal::SpinLock freelist_lock(
38 absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
39 ABSL_CONST_INIT static base_internal::ThreadIdentity* thread_identity_freelist;
40
41 // A per-thread destructor for reclaiming associated ThreadIdentity objects.
42 // Since we must preserve their storage we cache them for re-use.
ReclaimThreadIdentity(void * v)43 static void ReclaimThreadIdentity(void* v) {
44 base_internal::ThreadIdentity* identity =
45 static_cast<base_internal::ThreadIdentity*>(v);
46
47 // all_locks might have been allocated by the Mutex implementation.
48 // We free it here when we are notified that our thread is dying.
49 if (identity->per_thread_synch.all_locks != nullptr) {
50 base_internal::LowLevelAlloc::Free(identity->per_thread_synch.all_locks);
51 }
52
53 // We must explicitly clear the current thread's identity:
54 // (a) Subsequent (unrelated) per-thread destructors may require an identity.
55 // We must guarantee a new identity is used in this case (this instructor
56 // will be reinvoked up to PTHREAD_DESTRUCTOR_ITERATIONS in this case).
57 // (b) ThreadIdentity implementations may depend on memory that is not
58 // reinitialized before reuse. We must allow explicit clearing of the
59 // association state in this case.
60 base_internal::ClearCurrentThreadIdentity();
61 {
62 base_internal::SpinLockHolder l(&freelist_lock);
63 identity->next = thread_identity_freelist;
64 thread_identity_freelist = identity;
65 }
66 }
67
68 // Return value rounded up to next multiple of align.
69 // Align must be a power of two.
RoundUp(intptr_t addr,intptr_t align)70 static intptr_t RoundUp(intptr_t addr, intptr_t align) {
71 return (addr + align - 1) & ~(align - 1);
72 }
73
OneTimeInitThreadIdentity(base_internal::ThreadIdentity * identity)74 void OneTimeInitThreadIdentity(base_internal::ThreadIdentity* identity) {
75 PerThreadSem::Init(identity);
76 identity->ticker.store(0, std::memory_order_relaxed);
77 identity->wait_start.store(0, std::memory_order_relaxed);
78 identity->is_idle.store(false, std::memory_order_relaxed);
79 }
80
ResetThreadIdentityBetweenReuse(base_internal::ThreadIdentity * identity)81 static void ResetThreadIdentityBetweenReuse(
82 base_internal::ThreadIdentity* identity) {
83 base_internal::PerThreadSynch* pts = &identity->per_thread_synch;
84 pts->next = nullptr;
85 pts->skip = nullptr;
86 pts->may_skip = false;
87 pts->waitp = nullptr;
88 pts->suppress_fatal_errors = false;
89 pts->readers = 0;
90 pts->priority = 0;
91 pts->next_priority_read_cycles = 0;
92 pts->state.store(base_internal::PerThreadSynch::State::kAvailable,
93 std::memory_order_relaxed);
94 pts->maybe_unlocking = false;
95 pts->wake = false;
96 pts->cond_waiter = false;
97 pts->all_locks = nullptr;
98 identity->blocked_count_ptr = nullptr;
99 identity->ticker.store(0, std::memory_order_relaxed);
100 identity->wait_start.store(0, std::memory_order_relaxed);
101 identity->is_idle.store(false, std::memory_order_relaxed);
102 identity->next = nullptr;
103 }
104
NewThreadIdentity()105 static base_internal::ThreadIdentity* NewThreadIdentity() {
106 base_internal::ThreadIdentity* identity = nullptr;
107
108 {
109 // Re-use a previously released object if possible.
110 base_internal::SpinLockHolder l(&freelist_lock);
111 if (thread_identity_freelist) {
112 identity = thread_identity_freelist; // Take list-head.
113 thread_identity_freelist = thread_identity_freelist->next;
114 }
115 }
116
117 if (identity == nullptr) {
118 // Allocate enough space to align ThreadIdentity to a multiple of
119 // PerThreadSynch::kAlignment. This space is never released (it is
120 // added to a freelist by ReclaimThreadIdentity instead).
121 void* allocation = base_internal::LowLevelAlloc::Alloc(
122 sizeof(*identity) + base_internal::PerThreadSynch::kAlignment - 1);
123 // Round up the address to the required alignment.
124 identity = reinterpret_cast<base_internal::ThreadIdentity*>(
125 RoundUp(reinterpret_cast<intptr_t>(allocation),
126 base_internal::PerThreadSynch::kAlignment));
127 OneTimeInitThreadIdentity(identity);
128 }
129 ResetThreadIdentityBetweenReuse(identity);
130
131 return identity;
132 }
133
134 // Allocates and attaches ThreadIdentity object for the calling thread. Returns
135 // the new identity.
136 // REQUIRES: CurrentThreadIdentity(false) == nullptr
CreateThreadIdentity()137 base_internal::ThreadIdentity* CreateThreadIdentity() {
138 base_internal::ThreadIdentity* identity = NewThreadIdentity();
139 // Associate the value with the current thread, and attach our destructor.
140 base_internal::SetCurrentThreadIdentity(identity, ReclaimThreadIdentity);
141 return identity;
142 }
143
144 } // namespace synchronization_internal
145 ABSL_NAMESPACE_END
146 } // namespace absl
147
148 #endif // ABSL_LOW_LEVEL_ALLOC_MISSING
149