1 //===-- memtag.h ------------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef SCUDO_MEMTAG_H_
10 #define SCUDO_MEMTAG_H_
11
12 #include "internal_defs.h"
13
14 #if SCUDO_CAN_USE_MTE
15 #include <sys/auxv.h>
16 #include <sys/prctl.h>
17 #endif
18
19 namespace scudo {
20
21 #if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) || \
22 defined(SCUDO_FUZZ)
23
24 // We assume that Top-Byte Ignore is enabled if the architecture supports memory
25 // tagging. Not all operating systems enable TBI, so we only claim architectural
26 // support for memory tagging if the operating system enables TBI.
27 // HWASan uses the top byte for its own purpose and Scudo should not touch it.
28 #if SCUDO_CAN_USE_MTE && !defined(SCUDO_DISABLE_TBI) && \
29 !__has_feature(hwaddress_sanitizer)
archSupportsMemoryTagging()30 inline constexpr bool archSupportsMemoryTagging() { return true; }
31 #else
32 inline constexpr bool archSupportsMemoryTagging() { return false; }
33 #endif
34
archMemoryTagGranuleSize()35 inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
36
untagPointer(uptr Ptr)37 inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
38
extractTag(uptr Ptr)39 inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
40
41 #else
42
43 inline constexpr bool archSupportsMemoryTagging() { return false; }
44
45 inline NORETURN uptr archMemoryTagGranuleSize() {
46 UNREACHABLE("memory tagging not supported");
47 }
48
49 inline NORETURN uptr untagPointer(uptr Ptr) {
50 (void)Ptr;
51 UNREACHABLE("memory tagging not supported");
52 }
53
54 inline NORETURN uint8_t extractTag(uptr Ptr) {
55 (void)Ptr;
56 UNREACHABLE("memory tagging not supported");
57 }
58
59 #endif
60
61 #if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)
62
63 #if SCUDO_CAN_USE_MTE
64
systemSupportsMemoryTagging()65 inline bool systemSupportsMemoryTagging() {
66 #ifndef HWCAP2_MTE
67 #define HWCAP2_MTE (1 << 18)
68 #endif
69 return getauxval(AT_HWCAP2) & HWCAP2_MTE;
70 }
71
systemDetectsMemoryTagFaultsTestOnly()72 inline bool systemDetectsMemoryTagFaultsTestOnly() {
73 #ifndef PR_SET_TAGGED_ADDR_CTRL
74 #define PR_SET_TAGGED_ADDR_CTRL 54
75 #endif
76 #ifndef PR_GET_TAGGED_ADDR_CTRL
77 #define PR_GET_TAGGED_ADDR_CTRL 56
78 #endif
79 #ifndef PR_TAGGED_ADDR_ENABLE
80 #define PR_TAGGED_ADDR_ENABLE (1UL << 0)
81 #endif
82 #ifndef PR_MTE_TCF_SHIFT
83 #define PR_MTE_TCF_SHIFT 1
84 #endif
85 #ifndef PR_MTE_TAG_SHIFT
86 #define PR_MTE_TAG_SHIFT 3
87 #endif
88 #ifndef PR_MTE_TCF_NONE
89 #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
90 #endif
91 #ifndef PR_MTE_TCF_SYNC
92 #define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
93 #endif
94 #ifndef PR_MTE_TCF_MASK
95 #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
96 #endif
97 int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
98 if (res == -1)
99 return false;
100 return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
101 }
102
enableSystemMemoryTaggingTestOnly()103 inline void enableSystemMemoryTaggingTestOnly() {
104 prctl(PR_SET_TAGGED_ADDR_CTRL,
105 PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
106 0, 0, 0);
107 }
108
109 #else // !SCUDO_CAN_USE_MTE
110
systemSupportsMemoryTagging()111 inline bool systemSupportsMemoryTagging() { return false; }
112
systemDetectsMemoryTagFaultsTestOnly()113 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
114 UNREACHABLE("memory tagging not supported");
115 }
116
enableSystemMemoryTaggingTestOnly()117 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
118 UNREACHABLE("memory tagging not supported");
119 }
120
121 #endif // SCUDO_CAN_USE_MTE
122
123 class ScopedDisableMemoryTagChecks {
124 uptr PrevTCO;
125 bool active;
126
127 public:
active(cond)128 ScopedDisableMemoryTagChecks(bool cond = true) : active(cond) {
129 if (!active)
130 return;
131 __asm__ __volatile__(
132 R"(
133 .arch_extension memtag
134 mrs %0, tco
135 msr tco, #1
136 )"
137 : "=r"(PrevTCO));
138 }
139
~ScopedDisableMemoryTagChecks()140 ~ScopedDisableMemoryTagChecks() {
141 if (!active)
142 return;
143 __asm__ __volatile__(
144 R"(
145 .arch_extension memtag
146 msr tco, %0
147 )"
148 :
149 : "r"(PrevTCO));
150 }
151 };
152
selectRandomTag(uptr Ptr,uptr ExcludeMask)153 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
154 ExcludeMask |= 1; // Always exclude Tag 0.
155 uptr TaggedPtr;
156 __asm__ __volatile__(
157 R"(
158 .arch_extension memtag
159 irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
160 )"
161 : [TaggedPtr] "=r"(TaggedPtr)
162 : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
163 return TaggedPtr;
164 }
165
addFixedTag(uptr Ptr,uptr Tag)166 inline uptr addFixedTag(uptr Ptr, uptr Tag) {
167 DCHECK_LT(Tag, 16);
168 DCHECK_EQ(untagPointer(Ptr), Ptr);
169 return Ptr | (Tag << 56);
170 }
171
storeTags(uptr Begin,uptr End)172 inline uptr storeTags(uptr Begin, uptr End) {
173 DCHECK_EQ(0, Begin % 16);
174 uptr LineSize, Next, Tmp;
175 __asm__ __volatile__(
176 R"(
177 .arch_extension memtag
178
179 // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
180 // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
181 // indicates that the DC instructions are unavailable.
182 DCZID .req %[Tmp]
183 mrs DCZID, dczid_el0
184 tbnz DCZID, #4, 3f
185 and DCZID, DCZID, #15
186 mov %[LineSize], #4
187 lsl %[LineSize], %[LineSize], DCZID
188 .unreq DCZID
189
190 // Our main loop doesn't handle the case where we don't need to perform any
191 // DC GZVA operations. If the size of our tagged region is less than
192 // twice the cache line size, bail out to the slow path since it's not
193 // guaranteed that we'll be able to do a DC GZVA.
194 Size .req %[Tmp]
195 sub Size, %[End], %[Cur]
196 cmp Size, %[LineSize], lsl #1
197 b.lt 3f
198 .unreq Size
199
200 LineMask .req %[Tmp]
201 sub LineMask, %[LineSize], #1
202
203 // STZG until the start of the next cache line.
204 orr %[Next], %[Cur], LineMask
205 1:
206 stzg %[Cur], [%[Cur]], #16
207 cmp %[Cur], %[Next]
208 b.lt 1b
209
210 // DC GZVA cache lines until we have no more full cache lines.
211 bic %[Next], %[End], LineMask
212 .unreq LineMask
213 2:
214 dc gzva, %[Cur]
215 add %[Cur], %[Cur], %[LineSize]
216 cmp %[Cur], %[Next]
217 b.lt 2b
218
219 // STZG until the end of the tagged region. This loop is also used to handle
220 // slow path cases.
221 3:
222 cmp %[Cur], %[End]
223 b.ge 4f
224 stzg %[Cur], [%[Cur]], #16
225 b 3b
226
227 4:
228 )"
229 : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
230 [Tmp] "=&r"(Tmp)
231 : [End] "r"(End)
232 : "memory");
233 DCHECK_EQ(0, Begin % 16);
234 return Begin;
235 }
236
storeTag(uptr Ptr)237 inline void storeTag(uptr Ptr) {
238 DCHECK_EQ(0, Ptr % 16);
239 __asm__ __volatile__(R"(
240 .arch_extension memtag
241 stg %0, [%0]
242 )"
243 :
244 : "r"(Ptr)
245 : "memory");
246 }
247
loadTag(uptr Ptr)248 inline uptr loadTag(uptr Ptr) {
249 DCHECK_EQ(0, Ptr % 16);
250 uptr TaggedPtr = Ptr;
251 __asm__ __volatile__(
252 R"(
253 .arch_extension memtag
254 ldg %0, [%0]
255 )"
256 : "+r"(TaggedPtr)
257 :
258 : "memory");
259 return TaggedPtr;
260 }
261
262 #else
263
systemSupportsMemoryTagging()264 inline NORETURN bool systemSupportsMemoryTagging() {
265 UNREACHABLE("memory tagging not supported");
266 }
267
systemDetectsMemoryTagFaultsTestOnly()268 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
269 UNREACHABLE("memory tagging not supported");
270 }
271
enableSystemMemoryTaggingTestOnly()272 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
273 UNREACHABLE("memory tagging not supported");
274 }
275
276 struct ScopedDisableMemoryTagChecks {
277 ScopedDisableMemoryTagChecks(UNUSED bool cond = true) {}
278 };
279
selectRandomTag(uptr Ptr,uptr ExcludeMask)280 inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
281 (void)Ptr;
282 (void)ExcludeMask;
283 UNREACHABLE("memory tagging not supported");
284 }
285
addFixedTag(uptr Ptr,uptr Tag)286 inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) {
287 (void)Ptr;
288 (void)Tag;
289 UNREACHABLE("memory tagging not supported");
290 }
291
storeTags(uptr Begin,uptr End)292 inline NORETURN uptr storeTags(uptr Begin, uptr End) {
293 (void)Begin;
294 (void)End;
295 UNREACHABLE("memory tagging not supported");
296 }
297
storeTag(uptr Ptr)298 inline NORETURN void storeTag(uptr Ptr) {
299 (void)Ptr;
300 UNREACHABLE("memory tagging not supported");
301 }
302
loadTag(uptr Ptr)303 inline NORETURN uptr loadTag(uptr Ptr) {
304 (void)Ptr;
305 UNREACHABLE("memory tagging not supported");
306 }
307
308 #endif
309
310 #pragma GCC diagnostic push
311 #pragma GCC diagnostic ignored "-Wmissing-noreturn"
setRandomTag(void * Ptr,uptr Size,uptr ExcludeMask,uptr * TaggedBegin,uptr * TaggedEnd)312 inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
313 uptr *TaggedBegin, uptr *TaggedEnd) {
314 *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
315 *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
316 }
317 #pragma GCC diagnostic pop
318
untagPointer(void * Ptr)319 inline void *untagPointer(void *Ptr) {
320 return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr)));
321 }
322
loadTag(void * Ptr)323 inline void *loadTag(void *Ptr) {
324 return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr)));
325 }
326
addFixedTag(void * Ptr,uptr Tag)327 inline void *addFixedTag(void *Ptr, uptr Tag) {
328 return reinterpret_cast<void *>(
329 addFixedTag(reinterpret_cast<uptr>(Ptr), Tag));
330 }
331
332 template <typename Config>
allocatorSupportsMemoryTagging()333 inline constexpr bool allocatorSupportsMemoryTagging() {
334 return archSupportsMemoryTagging() && Config::getMaySupportMemoryTagging() &&
335 (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
336 }
337
338 } // namespace scudo
339
340 #endif
341