1 //===-- memtag.h ------------------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef SCUDO_MEMTAG_H_
10 #define SCUDO_MEMTAG_H_
11 
12 #include "internal_defs.h"
13 
14 #if SCUDO_CAN_USE_MTE
15 #include <sys/auxv.h>
16 #include <sys/prctl.h>
17 #endif
18 
19 namespace scudo {
20 
21 #if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) ||  \
22     defined(SCUDO_FUZZ)
23 
24 // We assume that Top-Byte Ignore is enabled if the architecture supports memory
25 // tagging. Not all operating systems enable TBI, so we only claim architectural
26 // support for memory tagging if the operating system enables TBI.
27 // HWASan uses the top byte for its own purpose and Scudo should not touch it.
28 #if SCUDO_CAN_USE_MTE && !defined(SCUDO_DISABLE_TBI) &&                        \
29     !__has_feature(hwaddress_sanitizer)
30 inline constexpr bool archSupportsMemoryTagging() { return true; }
31 #else
32 inline constexpr bool archSupportsMemoryTagging() { return false; }
33 #endif
34 
35 inline constexpr uptr archMemoryTagGranuleSize() { return 16; }
36 
37 inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); }
38 
39 inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; }
40 
41 #else
42 
43 inline constexpr bool archSupportsMemoryTagging() { return false; }
44 
45 inline NORETURN uptr archMemoryTagGranuleSize() {
46   UNREACHABLE("memory tagging not supported");
47 }
48 
49 inline NORETURN uptr untagPointer(uptr Ptr) {
50   (void)Ptr;
51   UNREACHABLE("memory tagging not supported");
52 }
53 
54 inline NORETURN uint8_t extractTag(uptr Ptr) {
55   (void)Ptr;
56   UNREACHABLE("memory tagging not supported");
57 }
58 
59 #endif
60 
61 #if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)
62 
63 #if SCUDO_CAN_USE_MTE
64 
65 inline bool systemSupportsMemoryTagging() {
66 #ifndef HWCAP2_MTE
67 #define HWCAP2_MTE (1 << 18)
68 #endif
69   return getauxval(AT_HWCAP2) & HWCAP2_MTE;
70 }
71 
72 inline bool systemDetectsMemoryTagFaultsTestOnly() {
73 #ifndef PR_SET_TAGGED_ADDR_CTRL
74 #define PR_SET_TAGGED_ADDR_CTRL 54
75 #endif
76 #ifndef PR_GET_TAGGED_ADDR_CTRL
77 #define PR_GET_TAGGED_ADDR_CTRL 56
78 #endif
79 #ifndef PR_TAGGED_ADDR_ENABLE
80 #define PR_TAGGED_ADDR_ENABLE (1UL << 0)
81 #endif
82 #ifndef PR_MTE_TCF_SHIFT
83 #define PR_MTE_TCF_SHIFT 1
84 #endif
85 #ifndef PR_MTE_TAG_SHIFT
86 #define PR_MTE_TAG_SHIFT 3
87 #endif
88 #ifndef PR_MTE_TCF_NONE
89 #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
90 #endif
91 #ifndef PR_MTE_TCF_SYNC
92 #define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
93 #endif
94 #ifndef PR_MTE_TCF_MASK
95 #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
96 #endif
97   int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
98   if (res == -1)
99     return false;
100   return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE;
101 }
102 
103 inline void enableSystemMemoryTaggingTestOnly() {
104   prctl(PR_SET_TAGGED_ADDR_CTRL,
105         PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
106         0, 0, 0);
107 }
108 
109 #else // !SCUDO_CAN_USE_MTE
110 
111 inline bool systemSupportsMemoryTagging() { return false; }
112 
113 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
114   UNREACHABLE("memory tagging not supported");
115 }
116 
117 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
118   UNREACHABLE("memory tagging not supported");
119 }
120 
121 #endif // SCUDO_CAN_USE_MTE
122 
123 class ScopedDisableMemoryTagChecks {
124   uptr PrevTCO;
125 
126 public:
127   ScopedDisableMemoryTagChecks() {
128     __asm__ __volatile__(
129         R"(
130         .arch_extension memtag
131         mrs %0, tco
132         msr tco, #1
133         )"
134         : "=r"(PrevTCO));
135   }
136 
137   ~ScopedDisableMemoryTagChecks() {
138     __asm__ __volatile__(
139         R"(
140         .arch_extension memtag
141         msr tco, %0
142         )"
143         :
144         : "r"(PrevTCO));
145   }
146 };
147 
148 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
149   ExcludeMask |= 1; // Always exclude Tag 0.
150   uptr TaggedPtr;
151   __asm__ __volatile__(
152       R"(
153       .arch_extension memtag
154       irg %[TaggedPtr], %[Ptr], %[ExcludeMask]
155       )"
156       : [TaggedPtr] "=r"(TaggedPtr)
157       : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask));
158   return TaggedPtr;
159 }
160 
161 inline uptr addFixedTag(uptr Ptr, uptr Tag) {
162   DCHECK_LT(Tag, 16);
163   DCHECK_EQ(untagPointer(Ptr), Ptr);
164   return Ptr | (Tag << 56);
165 }
166 
167 inline uptr storeTags(uptr Begin, uptr End) {
168   DCHECK_EQ(0, Begin % 16);
169   uptr LineSize, Next, Tmp;
170   __asm__ __volatile__(
171       R"(
172     .arch_extension memtag
173 
174     // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2
175     // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0
176     // indicates that the DC instructions are unavailable.
177     DCZID .req %[Tmp]
178     mrs DCZID, dczid_el0
179     tbnz DCZID, #4, 3f
180     and DCZID, DCZID, #15
181     mov %[LineSize], #4
182     lsl %[LineSize], %[LineSize], DCZID
183     .unreq DCZID
184 
185     // Our main loop doesn't handle the case where we don't need to perform any
186     // DC GZVA operations. If the size of our tagged region is less than
187     // twice the cache line size, bail out to the slow path since it's not
188     // guaranteed that we'll be able to do a DC GZVA.
189     Size .req %[Tmp]
190     sub Size, %[End], %[Cur]
191     cmp Size, %[LineSize], lsl #1
192     b.lt 3f
193     .unreq Size
194 
195     LineMask .req %[Tmp]
196     sub LineMask, %[LineSize], #1
197 
198     // STZG until the start of the next cache line.
199     orr %[Next], %[Cur], LineMask
200   1:
201     stzg %[Cur], [%[Cur]], #16
202     cmp %[Cur], %[Next]
203     b.lt 1b
204 
205     // DC GZVA cache lines until we have no more full cache lines.
206     bic %[Next], %[End], LineMask
207     .unreq LineMask
208   2:
209     dc gzva, %[Cur]
210     add %[Cur], %[Cur], %[LineSize]
211     cmp %[Cur], %[Next]
212     b.lt 2b
213 
214     // STZG until the end of the tagged region. This loop is also used to handle
215     // slow path cases.
216   3:
217     cmp %[Cur], %[End]
218     b.ge 4f
219     stzg %[Cur], [%[Cur]], #16
220     b 3b
221 
222   4:
223   )"
224       : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next),
225         [Tmp] "=&r"(Tmp)
226       : [End] "r"(End)
227       : "memory");
228   DCHECK_EQ(0, Begin % 16);
229   return Begin;
230 }
231 
232 inline void storeTag(uptr Ptr) {
233   DCHECK_EQ(0, Ptr % 16);
234   __asm__ __volatile__(R"(
235     .arch_extension memtag
236     stg %0, [%0]
237   )"
238                        :
239                        : "r"(Ptr)
240                        : "memory");
241 }
242 
243 inline uptr loadTag(uptr Ptr) {
244   DCHECK_EQ(0, Ptr % 16);
245   uptr TaggedPtr = Ptr;
246   __asm__ __volatile__(
247       R"(
248       .arch_extension memtag
249       ldg %0, [%0]
250       )"
251       : "+r"(TaggedPtr)
252       :
253       : "memory");
254   return TaggedPtr;
255 }
256 
257 #else
258 
259 inline NORETURN bool systemSupportsMemoryTagging() {
260   UNREACHABLE("memory tagging not supported");
261 }
262 
263 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() {
264   UNREACHABLE("memory tagging not supported");
265 }
266 
267 inline NORETURN void enableSystemMemoryTaggingTestOnly() {
268   UNREACHABLE("memory tagging not supported");
269 }
270 
271 struct ScopedDisableMemoryTagChecks {
272   ScopedDisableMemoryTagChecks() {}
273 };
274 
275 inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) {
276   (void)Ptr;
277   (void)ExcludeMask;
278   UNREACHABLE("memory tagging not supported");
279 }
280 
281 inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) {
282   (void)Ptr;
283   (void)Tag;
284   UNREACHABLE("memory tagging not supported");
285 }
286 
287 inline NORETURN uptr storeTags(uptr Begin, uptr End) {
288   (void)Begin;
289   (void)End;
290   UNREACHABLE("memory tagging not supported");
291 }
292 
293 inline NORETURN void storeTag(uptr Ptr) {
294   (void)Ptr;
295   UNREACHABLE("memory tagging not supported");
296 }
297 
298 inline NORETURN uptr loadTag(uptr Ptr) {
299   (void)Ptr;
300   UNREACHABLE("memory tagging not supported");
301 }
302 
303 #endif
304 
305 #pragma GCC diagnostic push
306 #pragma GCC diagnostic ignored "-Wmissing-noreturn"
307 inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask,
308                          uptr *TaggedBegin, uptr *TaggedEnd) {
309   *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask);
310   *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size);
311 }
312 #pragma GCC diagnostic pop
313 
314 inline void *untagPointer(void *Ptr) {
315   return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr)));
316 }
317 
318 inline void *loadTag(void *Ptr) {
319   return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr)));
320 }
321 
322 inline void *addFixedTag(void *Ptr, uptr Tag) {
323   return reinterpret_cast<void *>(
324       addFixedTag(reinterpret_cast<uptr>(Ptr), Tag));
325 }
326 
327 template <typename Config>
328 inline constexpr bool allocatorSupportsMemoryTagging() {
329   return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging &&
330          (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize();
331 }
332 
333 } // namespace scudo
334 
335 #endif
336