1 //===-- memtag.h ------------------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef SCUDO_MEMTAG_H_ 10 #define SCUDO_MEMTAG_H_ 11 12 #include "internal_defs.h" 13 14 #if SCUDO_LINUX 15 #include <sys/auxv.h> 16 #include <sys/prctl.h> 17 #endif 18 19 namespace scudo { 20 21 #if (__clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__)) || \ 22 defined(SCUDO_FUZZ) 23 24 // We assume that Top-Byte Ignore is enabled if the architecture supports memory 25 // tagging. Not all operating systems enable TBI, so we only claim architectural 26 // support for memory tagging if the operating system enables TBI. 27 // HWASan uses the top byte for its own purpose and Scudo should not touch it. 28 #if SCUDO_LINUX && !defined(SCUDO_DISABLE_TBI) && \ 29 !__has_feature(hwaddress_sanitizer) 30 inline constexpr bool archSupportsMemoryTagging() { return true; } 31 #else 32 inline constexpr bool archSupportsMemoryTagging() { return false; } 33 #endif 34 35 inline constexpr uptr archMemoryTagGranuleSize() { return 16; } 36 37 inline uptr untagPointer(uptr Ptr) { return Ptr & ((1ULL << 56) - 1); } 38 39 inline uint8_t extractTag(uptr Ptr) { return (Ptr >> 56) & 0xf; } 40 41 #else 42 43 inline constexpr bool archSupportsMemoryTagging() { return false; } 44 45 inline NORETURN uptr archMemoryTagGranuleSize() { 46 UNREACHABLE("memory tagging not supported"); 47 } 48 49 inline NORETURN uptr untagPointer(uptr Ptr) { 50 (void)Ptr; 51 UNREACHABLE("memory tagging not supported"); 52 } 53 54 inline NORETURN uint8_t extractTag(uptr Ptr) { 55 (void)Ptr; 56 UNREACHABLE("memory tagging not supported"); 57 } 58 59 #endif 60 61 #if __clang_major__ >= 12 && defined(__aarch64__) && !defined(__ILP32__) 62 63 #if SCUDO_LINUX 64 65 inline bool systemSupportsMemoryTagging() { 66 #ifndef HWCAP2_MTE 67 #define HWCAP2_MTE (1 << 18) 68 #endif 69 return getauxval(AT_HWCAP2) & HWCAP2_MTE; 70 } 71 72 inline bool systemDetectsMemoryTagFaultsTestOnly() { 73 #ifndef PR_SET_TAGGED_ADDR_CTRL 74 #define PR_SET_TAGGED_ADDR_CTRL 54 75 #endif 76 #ifndef PR_GET_TAGGED_ADDR_CTRL 77 #define PR_GET_TAGGED_ADDR_CTRL 56 78 #endif 79 #ifndef PR_TAGGED_ADDR_ENABLE 80 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) 81 #endif 82 #ifndef PR_MTE_TCF_SHIFT 83 #define PR_MTE_TCF_SHIFT 1 84 #endif 85 #ifndef PR_MTE_TAG_SHIFT 86 #define PR_MTE_TAG_SHIFT 3 87 #endif 88 #ifndef PR_MTE_TCF_NONE 89 #define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT) 90 #endif 91 #ifndef PR_MTE_TCF_SYNC 92 #define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT) 93 #endif 94 #ifndef PR_MTE_TCF_MASK 95 #define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT) 96 #endif 97 int res = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0); 98 if (res == -1) 99 return false; 100 return (static_cast<unsigned long>(res) & PR_MTE_TCF_MASK) != PR_MTE_TCF_NONE; 101 } 102 103 inline void enableSystemMemoryTaggingTestOnly() { 104 prctl(PR_SET_TAGGED_ADDR_CTRL, 105 PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT), 106 0, 0, 0); 107 } 108 109 #else // !SCUDO_LINUX 110 111 inline bool systemSupportsMemoryTagging() { return false; } 112 113 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() { 114 UNREACHABLE("memory tagging not supported"); 115 } 116 117 inline NORETURN void enableSystemMemoryTaggingTestOnly() { 118 UNREACHABLE("memory tagging not supported"); 119 } 120 121 #endif // SCUDO_LINUX 122 123 class ScopedDisableMemoryTagChecks { 124 uptr PrevTCO; 125 126 public: 127 ScopedDisableMemoryTagChecks() { 128 __asm__ __volatile__( 129 R"( 130 .arch_extension memtag 131 mrs %0, tco 132 msr tco, #1 133 )" 134 : "=r"(PrevTCO)); 135 } 136 137 ~ScopedDisableMemoryTagChecks() { 138 __asm__ __volatile__( 139 R"( 140 .arch_extension memtag 141 msr tco, %0 142 )" 143 : 144 : "r"(PrevTCO)); 145 } 146 }; 147 148 inline uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) { 149 ExcludeMask |= 1; // Always exclude Tag 0. 150 uptr TaggedPtr; 151 __asm__ __volatile__( 152 R"( 153 .arch_extension memtag 154 irg %[TaggedPtr], %[Ptr], %[ExcludeMask] 155 )" 156 : [TaggedPtr] "=r"(TaggedPtr) 157 : [Ptr] "r"(Ptr), [ExcludeMask] "r"(ExcludeMask)); 158 return TaggedPtr; 159 } 160 161 inline uptr addFixedTag(uptr Ptr, uptr Tag) { 162 DCHECK_LT(Tag, 16); 163 DCHECK_EQ(untagPointer(Ptr), Ptr); 164 return Ptr | (Tag << 56); 165 } 166 167 inline uptr storeTags(uptr Begin, uptr End) { 168 DCHECK_EQ(0, Begin % 16); 169 uptr LineSize, Next, Tmp; 170 __asm__ __volatile__( 171 R"( 172 .arch_extension memtag 173 174 // Compute the cache line size in bytes (DCZID_EL0 stores it as the log2 175 // of the number of 4-byte words) and bail out to the slow path if DCZID_EL0 176 // indicates that the DC instructions are unavailable. 177 DCZID .req %[Tmp] 178 mrs DCZID, dczid_el0 179 tbnz DCZID, #4, 3f 180 and DCZID, DCZID, #15 181 mov %[LineSize], #4 182 lsl %[LineSize], %[LineSize], DCZID 183 .unreq DCZID 184 185 // Our main loop doesn't handle the case where we don't need to perform any 186 // DC GZVA operations. If the size of our tagged region is less than 187 // twice the cache line size, bail out to the slow path since it's not 188 // guaranteed that we'll be able to do a DC GZVA. 189 Size .req %[Tmp] 190 sub Size, %[End], %[Cur] 191 cmp Size, %[LineSize], lsl #1 192 b.lt 3f 193 .unreq Size 194 195 LineMask .req %[Tmp] 196 sub LineMask, %[LineSize], #1 197 198 // STZG until the start of the next cache line. 199 orr %[Next], %[Cur], LineMask 200 1: 201 stzg %[Cur], [%[Cur]], #16 202 cmp %[Cur], %[Next] 203 b.lt 1b 204 205 // DC GZVA cache lines until we have no more full cache lines. 206 bic %[Next], %[End], LineMask 207 .unreq LineMask 208 2: 209 dc gzva, %[Cur] 210 add %[Cur], %[Cur], %[LineSize] 211 cmp %[Cur], %[Next] 212 b.lt 2b 213 214 // STZG until the end of the tagged region. This loop is also used to handle 215 // slow path cases. 216 3: 217 cmp %[Cur], %[End] 218 b.ge 4f 219 stzg %[Cur], [%[Cur]], #16 220 b 3b 221 222 4: 223 )" 224 : [Cur] "+&r"(Begin), [LineSize] "=&r"(LineSize), [Next] "=&r"(Next), 225 [Tmp] "=&r"(Tmp) 226 : [End] "r"(End) 227 : "memory"); 228 DCHECK_EQ(0, Begin % 16); 229 return Begin; 230 } 231 232 inline void storeTag(uptr Ptr) { 233 DCHECK_EQ(0, Ptr % 16); 234 __asm__ __volatile__(R"( 235 .arch_extension memtag 236 stg %0, [%0] 237 )" 238 : 239 : "r"(Ptr) 240 : "memory"); 241 } 242 243 inline uptr loadTag(uptr Ptr) { 244 DCHECK_EQ(0, Ptr % 16); 245 uptr TaggedPtr = Ptr; 246 __asm__ __volatile__( 247 R"( 248 .arch_extension memtag 249 ldg %0, [%0] 250 )" 251 : "+r"(TaggedPtr) 252 : 253 : "memory"); 254 return TaggedPtr; 255 } 256 257 #else 258 259 inline NORETURN bool systemSupportsMemoryTagging() { 260 UNREACHABLE("memory tagging not supported"); 261 } 262 263 inline NORETURN bool systemDetectsMemoryTagFaultsTestOnly() { 264 UNREACHABLE("memory tagging not supported"); 265 } 266 267 inline NORETURN void enableSystemMemoryTaggingTestOnly() { 268 UNREACHABLE("memory tagging not supported"); 269 } 270 271 struct ScopedDisableMemoryTagChecks { 272 ScopedDisableMemoryTagChecks() {} 273 }; 274 275 inline NORETURN uptr selectRandomTag(uptr Ptr, uptr ExcludeMask) { 276 (void)Ptr; 277 (void)ExcludeMask; 278 UNREACHABLE("memory tagging not supported"); 279 } 280 281 inline NORETURN uptr addFixedTag(uptr Ptr, uptr Tag) { 282 (void)Ptr; 283 (void)Tag; 284 UNREACHABLE("memory tagging not supported"); 285 } 286 287 inline NORETURN uptr storeTags(uptr Begin, uptr End) { 288 (void)Begin; 289 (void)End; 290 UNREACHABLE("memory tagging not supported"); 291 } 292 293 inline NORETURN void storeTag(uptr Ptr) { 294 (void)Ptr; 295 UNREACHABLE("memory tagging not supported"); 296 } 297 298 inline NORETURN uptr loadTag(uptr Ptr) { 299 (void)Ptr; 300 UNREACHABLE("memory tagging not supported"); 301 } 302 303 #endif 304 305 #pragma GCC diagnostic push 306 #pragma GCC diagnostic ignored "-Wmissing-noreturn" 307 inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, 308 uptr *TaggedBegin, uptr *TaggedEnd) { 309 *TaggedBegin = selectRandomTag(reinterpret_cast<uptr>(Ptr), ExcludeMask); 310 *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size); 311 } 312 #pragma GCC diagnostic pop 313 314 inline void *untagPointer(void *Ptr) { 315 return reinterpret_cast<void *>(untagPointer(reinterpret_cast<uptr>(Ptr))); 316 } 317 318 inline void *loadTag(void *Ptr) { 319 return reinterpret_cast<void *>(loadTag(reinterpret_cast<uptr>(Ptr))); 320 } 321 322 inline void *addFixedTag(void *Ptr, uptr Tag) { 323 return reinterpret_cast<void *>( 324 addFixedTag(reinterpret_cast<uptr>(Ptr), Tag)); 325 } 326 327 template <typename Config> 328 inline constexpr bool allocatorSupportsMemoryTagging() { 329 return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging && 330 (1 << SCUDO_MIN_ALIGNMENT_LOG) >= archMemoryTagGranuleSize(); 331 } 332 333 } // namespace scudo 334 335 #endif 336