1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Platform-specific code. 12 //===----------------------------------------------------------------------===// 13 14 #ifndef TSAN_PLATFORM_H 15 #define TSAN_PLATFORM_H 16 17 #if !defined(__LP64__) && !defined(_WIN64) 18 # error "Only 64-bit is supported" 19 #endif 20 21 #include "sanitizer_common/sanitizer_common.h" 22 #include "tsan_defs.h" 23 24 namespace __tsan { 25 26 enum { 27 // App memory is not mapped onto shadow memory range. 28 kBrokenMapping = 1 << 0, 29 // Mapping app memory and back does not produce the same address, 30 // this can lead to wrong addresses in reports and potentially 31 // other bad consequences. 32 kBrokenReverseMapping = 1 << 1, 33 // Mapping is non-linear for linear user range. 34 // This is bad and can lead to unpredictable memory corruptions, etc 35 // because range access functions assume linearity. 36 kBrokenLinearity = 1 << 2, 37 }; 38 39 /* 40 C/C++ on linux/x86_64 and freebsd/x86_64 41 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB) 42 0040 0000 0000 - 0100 0000 0000: - 43 0100 0000 0000 - 1000 0000 0000: shadow 44 1000 0000 0000 - 3000 0000 0000: - 45 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 46 4000 0000 0000 - 5500 0000 0000: - 47 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels 48 5680 0000 0000 - 7d00 0000 0000: - 49 7b00 0000 0000 - 7c00 0000 0000: heap 50 7c00 0000 0000 - 7e80 0000 0000: - 51 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack 52 53 C/C++ on netbsd/amd64 can reuse the same mapping: 54 * The address space starts from 0x1000 (option with 0x0) and ends with 55 0x7f7ffffff000. 56 * LoAppMem-kHeapMemEnd can be reused as it is. 57 * No VDSO support. 58 * No MidAppMem region. 59 * No additional HeapMem region. 60 * HiAppMem contains the stack, loader, shared libraries and heap. 61 * Stack on NetBSD/amd64 has prereserved 128MB. 62 * Heap grows downwards (top-down). 63 * ASLR must be disabled per-process or globally. 64 */ 65 struct Mapping48AddressSpace { 66 static const uptr kMetaShadowBeg = 0x300000000000ull; 67 static const uptr kMetaShadowEnd = 0x340000000000ull; 68 static const uptr kShadowBeg = 0x010000000000ull; 69 static const uptr kShadowEnd = 0x100000000000ull; 70 static const uptr kHeapMemBeg = 0x7b0000000000ull; 71 static const uptr kHeapMemEnd = 0x7c0000000000ull; 72 static const uptr kLoAppMemBeg = 0x000000001000ull; 73 static const uptr kLoAppMemEnd = 0x008000000000ull; 74 static const uptr kMidAppMemBeg = 0x550000000000ull; 75 static const uptr kMidAppMemEnd = 0x568000000000ull; 76 static const uptr kHiAppMemBeg = 0x7e8000000000ull; 77 static const uptr kHiAppMemEnd = 0x800000000000ull; 78 static const uptr kShadowMsk = 0x780000000000ull; 79 static const uptr kShadowXor = 0x040000000000ull; 80 static const uptr kShadowAdd = 0x000000000000ull; 81 static const uptr kVdsoBeg = 0xf000000000000000ull; 82 }; 83 84 /* 85 C/C++ on linux/mips64 (40-bit VMA) 86 0000 0000 00 - 0100 0000 00: - (4 GB) 87 0100 0000 00 - 0200 0000 00: main binary (4 GB) 88 0200 0000 00 - 1200 0000 00: - (64 GB) 89 1200 0000 00 - 2200 0000 00: shadow (64 GB) 90 2200 0000 00 - 4000 0000 00: - (120 GB) 91 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB) 92 5000 0000 00 - aa00 0000 00: - (360 GB) 93 aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB) 94 ab00 0000 00 - fe00 0000 00: - (332 GB) 95 fe00 0000 00 - ff00 0000 00: heap (4 GB) 96 ff00 0000 00 - ff80 0000 00: - (2 GB) 97 ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB) 98 */ 99 struct MappingMips64_40 { 100 static const uptr kMetaShadowBeg = 0x4000000000ull; 101 static const uptr kMetaShadowEnd = 0x5000000000ull; 102 static const uptr kShadowBeg = 0x1200000000ull; 103 static const uptr kShadowEnd = 0x2200000000ull; 104 static const uptr kHeapMemBeg = 0xfe00000000ull; 105 static const uptr kHeapMemEnd = 0xff00000000ull; 106 static const uptr kLoAppMemBeg = 0x0100000000ull; 107 static const uptr kLoAppMemEnd = 0x0200000000ull; 108 static const uptr kMidAppMemBeg = 0xaa00000000ull; 109 static const uptr kMidAppMemEnd = 0xab00000000ull; 110 static const uptr kHiAppMemBeg = 0xff80000000ull; 111 static const uptr kHiAppMemEnd = 0xffffffffffull; 112 static const uptr kShadowMsk = 0xf800000000ull; 113 static const uptr kShadowXor = 0x0800000000ull; 114 static const uptr kShadowAdd = 0x0000000000ull; 115 static const uptr kVdsoBeg = 0xfffff00000ull; 116 }; 117 118 /* 119 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) 120 0000 0000 00 - 0100 0000 00: - (4 GB) 121 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB) 122 0200 0000 00 - 0300 0000 00: heap (4 GB) 123 0300 0000 00 - 0400 0000 00: - (4 GB) 124 0400 0000 00 - 0800 0000 00: shadow memory (16 GB) 125 0800 0000 00 - 0d00 0000 00: - (20 GB) 126 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) 127 0e00 0000 00 - 1000 0000 00: - 128 */ 129 struct MappingAppleAarch64 { 130 static const uptr kLoAppMemBeg = 0x0100000000ull; 131 static const uptr kLoAppMemEnd = 0x0200000000ull; 132 static const uptr kHeapMemBeg = 0x0200000000ull; 133 static const uptr kHeapMemEnd = 0x0300000000ull; 134 static const uptr kShadowBeg = 0x0400000000ull; 135 static const uptr kShadowEnd = 0x0800000000ull; 136 static const uptr kMetaShadowBeg = 0x0d00000000ull; 137 static const uptr kMetaShadowEnd = 0x0e00000000ull; 138 static const uptr kHiAppMemBeg = 0x0fc0000000ull; 139 static const uptr kHiAppMemEnd = 0x0fc0000000ull; 140 static const uptr kShadowMsk = 0x0ull; 141 static const uptr kShadowXor = 0x0ull; 142 static const uptr kShadowAdd = 0x0200000000ull; 143 static const uptr kVdsoBeg = 0x7000000000000000ull; 144 static const uptr kMidAppMemBeg = 0; 145 static const uptr kMidAppMemEnd = 0; 146 }; 147 148 /* 149 C/C++ on linux/aarch64 (39-bit VMA) 150 0000 0010 00 - 0100 0000 00: main binary 151 0100 0000 00 - 0400 0000 00: - 152 0400 0000 00 - 1000 0000 00: shadow memory 153 2000 0000 00 - 3100 0000 00: - 154 3100 0000 00 - 3400 0000 00: metainfo 155 3400 0000 00 - 5500 0000 00: - 156 5500 0000 00 - 5600 0000 00: main binary (PIE) 157 5600 0000 00 - 7c00 0000 00: - 158 7c00 0000 00 - 7d00 0000 00: heap 159 7d00 0000 00 - 7fff ffff ff: modules and main thread stack 160 */ 161 struct MappingAarch64_39 { 162 static const uptr kLoAppMemBeg = 0x0000001000ull; 163 static const uptr kLoAppMemEnd = 0x0100000000ull; 164 static const uptr kShadowBeg = 0x0400000000ull; 165 static const uptr kShadowEnd = 0x1000000000ull; 166 static const uptr kMetaShadowBeg = 0x3100000000ull; 167 static const uptr kMetaShadowEnd = 0x3400000000ull; 168 static const uptr kMidAppMemBeg = 0x5500000000ull; 169 static const uptr kMidAppMemEnd = 0x5600000000ull; 170 static const uptr kHeapMemBeg = 0x7c00000000ull; 171 static const uptr kHeapMemEnd = 0x7d00000000ull; 172 static const uptr kHiAppMemBeg = 0x7e00000000ull; 173 static const uptr kHiAppMemEnd = 0x7fffffffffull; 174 static const uptr kShadowMsk = 0x7800000000ull; 175 static const uptr kShadowXor = 0x0200000000ull; 176 static const uptr kShadowAdd = 0x0000000000ull; 177 static const uptr kVdsoBeg = 0x7f00000000ull; 178 }; 179 180 /* 181 C/C++ on linux/aarch64 (42-bit VMA) 182 00000 0010 00 - 01000 0000 00: main binary 183 01000 0000 00 - 08000 0000 00: - 184 08000 0000 00 - 10000 0000 00: shadow memory 185 10000 0000 00 - 26000 0000 00: - 186 26000 0000 00 - 28000 0000 00: metainfo 187 28000 0000 00 - 2aa00 0000 00: - 188 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) 189 2ab00 0000 00 - 3e000 0000 00: - 190 3e000 0000 00 - 3f000 0000 00: heap 191 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack 192 */ 193 struct MappingAarch64_42 { 194 static const uptr kBroken = kBrokenReverseMapping; 195 static const uptr kLoAppMemBeg = 0x00000001000ull; 196 static const uptr kLoAppMemEnd = 0x01000000000ull; 197 static const uptr kShadowBeg = 0x08000000000ull; 198 static const uptr kShadowEnd = 0x10000000000ull; 199 static const uptr kMetaShadowBeg = 0x26000000000ull; 200 static const uptr kMetaShadowEnd = 0x28000000000ull; 201 static const uptr kMidAppMemBeg = 0x2aa00000000ull; 202 static const uptr kMidAppMemEnd = 0x2ab00000000ull; 203 static const uptr kHeapMemBeg = 0x3e000000000ull; 204 static const uptr kHeapMemEnd = 0x3f000000000ull; 205 static const uptr kHiAppMemBeg = 0x3f000000000ull; 206 static const uptr kHiAppMemEnd = 0x3ffffffffffull; 207 static const uptr kShadowMsk = 0x3c000000000ull; 208 static const uptr kShadowXor = 0x04000000000ull; 209 static const uptr kShadowAdd = 0x00000000000ull; 210 static const uptr kVdsoBeg = 0x37f00000000ull; 211 }; 212 213 struct MappingAarch64_48 { 214 static const uptr kLoAppMemBeg = 0x0000000001000ull; 215 static const uptr kLoAppMemEnd = 0x0000200000000ull; 216 static const uptr kShadowBeg = 0x0001000000000ull; 217 static const uptr kShadowEnd = 0x0002000000000ull; 218 static const uptr kMetaShadowBeg = 0x0005000000000ull; 219 static const uptr kMetaShadowEnd = 0x0006000000000ull; 220 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull; 221 static const uptr kMidAppMemEnd = 0x0aaaf00000000ull; 222 static const uptr kHeapMemBeg = 0x0ffff00000000ull; 223 static const uptr kHeapMemEnd = 0x0ffff00000000ull; 224 static const uptr kHiAppMemBeg = 0x0ffff00000000ull; 225 static const uptr kHiAppMemEnd = 0x1000000000000ull; 226 static const uptr kShadowMsk = 0x0fff800000000ull; 227 static const uptr kShadowXor = 0x0000800000000ull; 228 static const uptr kShadowAdd = 0x0000000000000ull; 229 static const uptr kVdsoBeg = 0xffff000000000ull; 230 }; 231 232 /* 233 C/C++ on linux/powerpc64 (44-bit VMA) 234 0000 0000 0100 - 0001 0000 0000: main binary 235 0001 0000 0000 - 0001 0000 0000: - 236 0001 0000 0000 - 0b00 0000 0000: shadow 237 0b00 0000 0000 - 0b00 0000 0000: - 238 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) 239 0d00 0000 0000 - 0f00 0000 0000: - 240 0f00 0000 0000 - 0f50 0000 0000: heap 241 0f50 0000 0000 - 0f60 0000 0000: - 242 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack 243 */ 244 struct MappingPPC64_44 { 245 static const uptr kBroken = 246 kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity; 247 static const uptr kMetaShadowBeg = 0x0b0000000000ull; 248 static const uptr kMetaShadowEnd = 0x0d0000000000ull; 249 static const uptr kShadowBeg = 0x000100000000ull; 250 static const uptr kShadowEnd = 0x0b0000000000ull; 251 static const uptr kLoAppMemBeg = 0x000000000100ull; 252 static const uptr kLoAppMemEnd = 0x000100000000ull; 253 static const uptr kHeapMemBeg = 0x0f0000000000ull; 254 static const uptr kHeapMemEnd = 0x0f5000000000ull; 255 static const uptr kHiAppMemBeg = 0x0f6000000000ull; 256 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits 257 static const uptr kShadowMsk = 0x0f0000000000ull; 258 static const uptr kShadowXor = 0x002100000000ull; 259 static const uptr kShadowAdd = 0x000000000000ull; 260 static const uptr kVdsoBeg = 0x3c0000000000000ull; 261 static const uptr kMidAppMemBeg = 0; 262 static const uptr kMidAppMemEnd = 0; 263 }; 264 265 /* 266 C/C++ on linux/powerpc64 (46-bit VMA) 267 0000 0000 1000 - 0100 0000 0000: main binary 268 0100 0000 0000 - 0200 0000 0000: - 269 0100 0000 0000 - 0800 0000 0000: shadow 270 0800 0000 0000 - 1000 0000 0000: - 271 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) 272 1200 0000 0000 - 3d00 0000 0000: - 273 3d00 0000 0000 - 3e00 0000 0000: heap 274 3e00 0000 0000 - 3e80 0000 0000: - 275 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack 276 */ 277 struct MappingPPC64_46 { 278 static const uptr kMetaShadowBeg = 0x100000000000ull; 279 static const uptr kMetaShadowEnd = 0x120000000000ull; 280 static const uptr kShadowBeg = 0x010000000000ull; 281 static const uptr kShadowEnd = 0x080000000000ull; 282 static const uptr kHeapMemBeg = 0x3d0000000000ull; 283 static const uptr kHeapMemEnd = 0x3e0000000000ull; 284 static const uptr kLoAppMemBeg = 0x000000001000ull; 285 static const uptr kLoAppMemEnd = 0x010000000000ull; 286 static const uptr kHiAppMemBeg = 0x3e8000000000ull; 287 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits 288 static const uptr kShadowMsk = 0x3c0000000000ull; 289 static const uptr kShadowXor = 0x020000000000ull; 290 static const uptr kShadowAdd = 0x000000000000ull; 291 static const uptr kVdsoBeg = 0x7800000000000000ull; 292 static const uptr kMidAppMemBeg = 0; 293 static const uptr kMidAppMemEnd = 0; 294 }; 295 296 /* 297 C/C++ on linux/powerpc64 (47-bit VMA) 298 0000 0000 1000 - 0100 0000 0000: main binary 299 0100 0000 0000 - 0200 0000 0000: - 300 0100 0000 0000 - 0800 0000 0000: shadow 301 0800 0000 0000 - 1000 0000 0000: - 302 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) 303 1200 0000 0000 - 7d00 0000 0000: - 304 7d00 0000 0000 - 7e00 0000 0000: heap 305 7e00 0000 0000 - 7e80 0000 0000: - 306 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack 307 */ 308 struct MappingPPC64_47 { 309 static const uptr kMetaShadowBeg = 0x100000000000ull; 310 static const uptr kMetaShadowEnd = 0x120000000000ull; 311 static const uptr kShadowBeg = 0x010000000000ull; 312 static const uptr kShadowEnd = 0x080000000000ull; 313 static const uptr kHeapMemBeg = 0x7d0000000000ull; 314 static const uptr kHeapMemEnd = 0x7e0000000000ull; 315 static const uptr kLoAppMemBeg = 0x000000001000ull; 316 static const uptr kLoAppMemEnd = 0x010000000000ull; 317 static const uptr kHiAppMemBeg = 0x7e8000000000ull; 318 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits 319 static const uptr kShadowMsk = 0x7c0000000000ull; 320 static const uptr kShadowXor = 0x020000000000ull; 321 static const uptr kShadowAdd = 0x000000000000ull; 322 static const uptr kVdsoBeg = 0x7800000000000000ull; 323 static const uptr kMidAppMemBeg = 0; 324 static const uptr kMidAppMemEnd = 0; 325 }; 326 327 /* 328 C/C++ on linux/s390x 329 While the kernel provides a 64-bit address space, we have to restrict ourselves 330 to 48 bits due to how e.g. SyncVar::GetId() works. 331 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB 332 0e00 0000 0000 - 2000 0000 0000: - 333 2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app) 334 4000 0000 0000 - 9000 0000 0000: - 335 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) 336 9800 0000 0000 - be00 0000 0000: - 337 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator) 338 */ 339 struct MappingS390x { 340 static const uptr kMetaShadowBeg = 0x900000000000ull; 341 static const uptr kMetaShadowEnd = 0x980000000000ull; 342 static const uptr kShadowBeg = 0x200000000000ull; 343 static const uptr kShadowEnd = 0x400000000000ull; 344 static const uptr kHeapMemBeg = 0xbe0000000000ull; 345 static const uptr kHeapMemEnd = 0xc00000000000ull; 346 static const uptr kLoAppMemBeg = 0x000000001000ull; 347 static const uptr kLoAppMemEnd = 0x0e0000000000ull; 348 static const uptr kHiAppMemBeg = 0xc00000004000ull; 349 static const uptr kHiAppMemEnd = 0xc00000004000ull; 350 static const uptr kShadowMsk = 0xb00000000000ull; 351 static const uptr kShadowXor = 0x100000000000ull; 352 static const uptr kShadowAdd = 0x000000000000ull; 353 static const uptr kVdsoBeg = 0xfffffffff000ull; 354 static const uptr kMidAppMemBeg = 0; 355 static const uptr kMidAppMemEnd = 0; 356 }; 357 358 /* Go on linux, darwin and freebsd on x86_64 359 0000 0000 1000 - 0000 1000 0000: executable 360 0000 1000 0000 - 00c0 0000 0000: - 361 00c0 0000 0000 - 00e0 0000 0000: heap 362 00e0 0000 0000 - 2000 0000 0000: - 363 2000 0000 0000 - 21c0 0000 0000: shadow 364 21c0 0000 0000 - 3000 0000 0000: - 365 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 366 4000 0000 0000 - 8000 0000 0000: - 367 */ 368 369 struct MappingGo48 { 370 static const uptr kMetaShadowBeg = 0x300000000000ull; 371 static const uptr kMetaShadowEnd = 0x400000000000ull; 372 static const uptr kShadowBeg = 0x200000000000ull; 373 static const uptr kShadowEnd = 0x21c000000000ull; 374 static const uptr kLoAppMemBeg = 0x000000001000ull; 375 static const uptr kLoAppMemEnd = 0x00e000000000ull; 376 static const uptr kMidAppMemBeg = 0; 377 static const uptr kMidAppMemEnd = 0; 378 static const uptr kHiAppMemBeg = 0; 379 static const uptr kHiAppMemEnd = 0; 380 static const uptr kHeapMemBeg = 0; 381 static const uptr kHeapMemEnd = 0; 382 static const uptr kVdsoBeg = 0; 383 static const uptr kShadowMsk = 0; 384 static const uptr kShadowXor = 0; 385 static const uptr kShadowAdd = 0x200000000000ull; 386 }; 387 388 /* Go on windows 389 0000 0000 1000 - 0000 1000 0000: executable 390 0000 1000 0000 - 00f8 0000 0000: - 391 00c0 0000 0000 - 00e0 0000 0000: heap 392 00e0 0000 0000 - 0100 0000 0000: - 393 0100 0000 0000 - 0300 0000 0000: shadow 394 0300 0000 0000 - 0700 0000 0000: - 395 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects) 396 07d0 0000 0000 - 8000 0000 0000: - 397 PIE binaries currently not supported, but it should be theoretically possible. 398 */ 399 400 struct MappingGoWindows { 401 static const uptr kMetaShadowBeg = 0x070000000000ull; 402 static const uptr kMetaShadowEnd = 0x077000000000ull; 403 static const uptr kShadowBeg = 0x010000000000ull; 404 static const uptr kShadowEnd = 0x030000000000ull; 405 static const uptr kLoAppMemBeg = 0x000000001000ull; 406 static const uptr kLoAppMemEnd = 0x00e000000000ull; 407 static const uptr kMidAppMemBeg = 0; 408 static const uptr kMidAppMemEnd = 0; 409 static const uptr kHiAppMemBeg = 0; 410 static const uptr kHiAppMemEnd = 0; 411 static const uptr kHeapMemBeg = 0; 412 static const uptr kHeapMemEnd = 0; 413 static const uptr kVdsoBeg = 0; 414 static const uptr kShadowMsk = 0; 415 static const uptr kShadowXor = 0; 416 static const uptr kShadowAdd = 0x010000000000ull; 417 }; 418 419 /* Go on linux/powerpc64 (46-bit VMA) 420 0000 0000 1000 - 0000 1000 0000: executable 421 0000 1000 0000 - 00c0 0000 0000: - 422 00c0 0000 0000 - 00e0 0000 0000: heap 423 00e0 0000 0000 - 2000 0000 0000: - 424 2000 0000 0000 - 21c0 0000 0000: shadow 425 21c0 0000 0000 - 2400 0000 0000: - 426 2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects) 427 2470 0000 0000 - 4000 0000 0000: - 428 */ 429 430 struct MappingGoPPC64_46 { 431 static const uptr kMetaShadowBeg = 0x240000000000ull; 432 static const uptr kMetaShadowEnd = 0x247000000000ull; 433 static const uptr kShadowBeg = 0x200000000000ull; 434 static const uptr kShadowEnd = 0x21c000000000ull; 435 static const uptr kLoAppMemBeg = 0x000000001000ull; 436 static const uptr kLoAppMemEnd = 0x00e000000000ull; 437 static const uptr kMidAppMemBeg = 0; 438 static const uptr kMidAppMemEnd = 0; 439 static const uptr kHiAppMemBeg = 0; 440 static const uptr kHiAppMemEnd = 0; 441 static const uptr kHeapMemBeg = 0; 442 static const uptr kHeapMemEnd = 0; 443 static const uptr kVdsoBeg = 0; 444 static const uptr kShadowMsk = 0; 445 static const uptr kShadowXor = 0; 446 static const uptr kShadowAdd = 0x200000000000ull; 447 }; 448 449 /* Go on linux/powerpc64 (47-bit VMA) 450 0000 0000 1000 - 0000 1000 0000: executable 451 0000 1000 0000 - 00c0 0000 0000: - 452 00c0 0000 0000 - 00e0 0000 0000: heap 453 00e0 0000 0000 - 2000 0000 0000: - 454 2000 0000 0000 - 2800 0000 0000: shadow 455 2800 0000 0000 - 3000 0000 0000: - 456 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 457 3200 0000 0000 - 8000 0000 0000: - 458 */ 459 460 struct MappingGoPPC64_47 { 461 static const uptr kMetaShadowBeg = 0x300000000000ull; 462 static const uptr kMetaShadowEnd = 0x320000000000ull; 463 static const uptr kShadowBeg = 0x200000000000ull; 464 static const uptr kShadowEnd = 0x280000000000ull; 465 static const uptr kLoAppMemBeg = 0x000000001000ull; 466 static const uptr kLoAppMemEnd = 0x00e000000000ull; 467 static const uptr kMidAppMemBeg = 0; 468 static const uptr kMidAppMemEnd = 0; 469 static const uptr kHiAppMemBeg = 0; 470 static const uptr kHiAppMemEnd = 0; 471 static const uptr kHeapMemBeg = 0; 472 static const uptr kHeapMemEnd = 0; 473 static const uptr kVdsoBeg = 0; 474 static const uptr kShadowMsk = 0; 475 static const uptr kShadowXor = 0; 476 static const uptr kShadowAdd = 0x200000000000ull; 477 }; 478 479 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA) 480 0000 0000 1000 - 0000 1000 0000: executable 481 0000 1000 0000 - 00c0 0000 0000: - 482 00c0 0000 0000 - 00e0 0000 0000: heap 483 00e0 0000 0000 - 2000 0000 0000: - 484 2000 0000 0000 - 2800 0000 0000: shadow 485 2800 0000 0000 - 3000 0000 0000: - 486 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 487 3200 0000 0000 - 8000 0000 0000: - 488 */ 489 struct MappingGoAarch64 { 490 static const uptr kMetaShadowBeg = 0x300000000000ull; 491 static const uptr kMetaShadowEnd = 0x320000000000ull; 492 static const uptr kShadowBeg = 0x200000000000ull; 493 static const uptr kShadowEnd = 0x280000000000ull; 494 static const uptr kLoAppMemBeg = 0x000000001000ull; 495 static const uptr kLoAppMemEnd = 0x00e000000000ull; 496 static const uptr kMidAppMemBeg = 0; 497 static const uptr kMidAppMemEnd = 0; 498 static const uptr kHiAppMemBeg = 0; 499 static const uptr kHiAppMemEnd = 0; 500 static const uptr kHeapMemBeg = 0; 501 static const uptr kHeapMemEnd = 0; 502 static const uptr kVdsoBeg = 0; 503 static const uptr kShadowMsk = 0; 504 static const uptr kShadowXor = 0; 505 static const uptr kShadowAdd = 0x200000000000ull; 506 }; 507 508 /* 509 Go on linux/mips64 (47-bit VMA) 510 0000 0000 1000 - 0000 1000 0000: executable 511 0000 1000 0000 - 00c0 0000 0000: - 512 00c0 0000 0000 - 00e0 0000 0000: heap 513 00e0 0000 0000 - 2000 0000 0000: - 514 2000 0000 0000 - 2800 0000 0000: shadow 515 2800 0000 0000 - 3000 0000 0000: - 516 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 517 3200 0000 0000 - 8000 0000 0000: - 518 */ 519 struct MappingGoMips64_47 { 520 static const uptr kMetaShadowBeg = 0x300000000000ull; 521 static const uptr kMetaShadowEnd = 0x320000000000ull; 522 static const uptr kShadowBeg = 0x200000000000ull; 523 static const uptr kShadowEnd = 0x280000000000ull; 524 static const uptr kLoAppMemBeg = 0x000000001000ull; 525 static const uptr kLoAppMemEnd = 0x00e000000000ull; 526 static const uptr kMidAppMemBeg = 0; 527 static const uptr kMidAppMemEnd = 0; 528 static const uptr kHiAppMemBeg = 0; 529 static const uptr kHiAppMemEnd = 0; 530 static const uptr kHeapMemBeg = 0; 531 static const uptr kHeapMemEnd = 0; 532 static const uptr kVdsoBeg = 0; 533 static const uptr kShadowMsk = 0; 534 static const uptr kShadowXor = 0; 535 static const uptr kShadowAdd = 0x200000000000ull; 536 }; 537 538 /* 539 Go on linux/s390x 540 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB 541 1000 0000 0000 - 4000 0000 0000: - 542 4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app) 543 6000 0000 0000 - 9000 0000 0000: - 544 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) 545 */ 546 struct MappingGoS390x { 547 static const uptr kMetaShadowBeg = 0x900000000000ull; 548 static const uptr kMetaShadowEnd = 0x980000000000ull; 549 static const uptr kShadowBeg = 0x400000000000ull; 550 static const uptr kShadowEnd = 0x600000000000ull; 551 static const uptr kLoAppMemBeg = 0x000000001000ull; 552 static const uptr kLoAppMemEnd = 0x100000000000ull; 553 static const uptr kMidAppMemBeg = 0; 554 static const uptr kMidAppMemEnd = 0; 555 static const uptr kHiAppMemBeg = 0; 556 static const uptr kHiAppMemEnd = 0; 557 static const uptr kHeapMemBeg = 0; 558 static const uptr kHeapMemEnd = 0; 559 static const uptr kVdsoBeg = 0; 560 static const uptr kShadowMsk = 0; 561 static const uptr kShadowXor = 0; 562 static const uptr kShadowAdd = 0x400000000000ull; 563 }; 564 565 extern uptr vmaSize; 566 567 template <typename Func, typename Arg> 568 ALWAYS_INLINE auto SelectMapping(Arg arg) { 569 #if SANITIZER_GO 570 # if defined(__powerpc64__) 571 switch (vmaSize) { 572 case 46: 573 return Func::template Apply<MappingGoPPC64_46>(arg); 574 case 47: 575 return Func::template Apply<MappingGoPPC64_47>(arg); 576 } 577 # elif defined(__mips64) 578 return Func::template Apply<MappingGoMips64_47>(arg); 579 # elif defined(__s390x__) 580 return Func::template Apply<MappingGoS390x>(arg); 581 # elif defined(__aarch64__) 582 return Func::template Apply<MappingGoAarch64>(arg); 583 # elif SANITIZER_WINDOWS 584 return Func::template Apply<MappingGoWindows>(arg); 585 # else 586 return Func::template Apply<MappingGo48>(arg); 587 # endif 588 #else // SANITIZER_GO 589 # if SANITIZER_IOS && !SANITIZER_IOSSIM 590 return Func::template Apply<MappingAppleAarch64>(arg); 591 # elif defined(__x86_64__) || SANITIZER_APPLE 592 return Func::template Apply<Mapping48AddressSpace>(arg); 593 # elif defined(__aarch64__) 594 switch (vmaSize) { 595 case 39: 596 return Func::template Apply<MappingAarch64_39>(arg); 597 case 42: 598 return Func::template Apply<MappingAarch64_42>(arg); 599 case 48: 600 return Func::template Apply<MappingAarch64_48>(arg); 601 } 602 # elif defined(__powerpc64__) 603 switch (vmaSize) { 604 case 44: 605 return Func::template Apply<MappingPPC64_44>(arg); 606 case 46: 607 return Func::template Apply<MappingPPC64_46>(arg); 608 case 47: 609 return Func::template Apply<MappingPPC64_47>(arg); 610 } 611 # elif defined(__mips64) 612 return Func::template Apply<MappingMips64_40>(arg); 613 # elif defined(__s390x__) 614 return Func::template Apply<MappingS390x>(arg); 615 # else 616 # error "unsupported platform" 617 # endif 618 #endif 619 Die(); 620 } 621 622 template <typename Func> 623 void ForEachMapping() { 624 Func::template Apply<Mapping48AddressSpace>(); 625 Func::template Apply<MappingMips64_40>(); 626 Func::template Apply<MappingAppleAarch64>(); 627 Func::template Apply<MappingAarch64_39>(); 628 Func::template Apply<MappingAarch64_42>(); 629 Func::template Apply<MappingAarch64_48>(); 630 Func::template Apply<MappingPPC64_44>(); 631 Func::template Apply<MappingPPC64_46>(); 632 Func::template Apply<MappingPPC64_47>(); 633 Func::template Apply<MappingS390x>(); 634 Func::template Apply<MappingGo48>(); 635 Func::template Apply<MappingGoWindows>(); 636 Func::template Apply<MappingGoPPC64_46>(); 637 Func::template Apply<MappingGoPPC64_47>(); 638 Func::template Apply<MappingGoAarch64>(); 639 Func::template Apply<MappingGoMips64_47>(); 640 Func::template Apply<MappingGoS390x>(); 641 } 642 643 enum MappingType { 644 kLoAppMemBeg, 645 kLoAppMemEnd, 646 kHiAppMemBeg, 647 kHiAppMemEnd, 648 kMidAppMemBeg, 649 kMidAppMemEnd, 650 kHeapMemBeg, 651 kHeapMemEnd, 652 kShadowBeg, 653 kShadowEnd, 654 kMetaShadowBeg, 655 kMetaShadowEnd, 656 kVdsoBeg, 657 }; 658 659 struct MappingField { 660 template <typename Mapping> 661 static uptr Apply(MappingType type) { 662 switch (type) { 663 case kLoAppMemBeg: 664 return Mapping::kLoAppMemBeg; 665 case kLoAppMemEnd: 666 return Mapping::kLoAppMemEnd; 667 case kMidAppMemBeg: 668 return Mapping::kMidAppMemBeg; 669 case kMidAppMemEnd: 670 return Mapping::kMidAppMemEnd; 671 case kHiAppMemBeg: 672 return Mapping::kHiAppMemBeg; 673 case kHiAppMemEnd: 674 return Mapping::kHiAppMemEnd; 675 case kHeapMemBeg: 676 return Mapping::kHeapMemBeg; 677 case kHeapMemEnd: 678 return Mapping::kHeapMemEnd; 679 case kVdsoBeg: 680 return Mapping::kVdsoBeg; 681 case kShadowBeg: 682 return Mapping::kShadowBeg; 683 case kShadowEnd: 684 return Mapping::kShadowEnd; 685 case kMetaShadowBeg: 686 return Mapping::kMetaShadowBeg; 687 case kMetaShadowEnd: 688 return Mapping::kMetaShadowEnd; 689 } 690 Die(); 691 } 692 }; 693 694 ALWAYS_INLINE 695 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); } 696 ALWAYS_INLINE 697 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); } 698 699 ALWAYS_INLINE 700 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); } 701 ALWAYS_INLINE 702 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); } 703 704 ALWAYS_INLINE 705 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); } 706 ALWAYS_INLINE 707 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); } 708 709 ALWAYS_INLINE 710 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); } 711 ALWAYS_INLINE 712 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); } 713 714 ALWAYS_INLINE 715 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); } 716 717 ALWAYS_INLINE 718 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); } 719 ALWAYS_INLINE 720 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); } 721 722 ALWAYS_INLINE 723 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); } 724 ALWAYS_INLINE 725 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); } 726 727 struct IsAppMemImpl { 728 template <typename Mapping> 729 static bool Apply(uptr mem) { 730 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || 731 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || 732 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || 733 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); 734 } 735 }; 736 737 ALWAYS_INLINE 738 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); } 739 740 struct IsShadowMemImpl { 741 template <typename Mapping> 742 static bool Apply(uptr mem) { 743 return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; 744 } 745 }; 746 747 ALWAYS_INLINE 748 bool IsShadowMem(RawShadow *p) { 749 return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p)); 750 } 751 752 struct IsMetaMemImpl { 753 template <typename Mapping> 754 static bool Apply(uptr mem) { 755 return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; 756 } 757 }; 758 759 ALWAYS_INLINE 760 bool IsMetaMem(const u32 *p) { 761 return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p)); 762 } 763 764 struct MemToShadowImpl { 765 template <typename Mapping> 766 static uptr Apply(uptr x) { 767 DCHECK(IsAppMemImpl::Apply<Mapping>(x)); 768 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^ 769 Mapping::kShadowXor) * 770 kShadowMultiplier + 771 Mapping::kShadowAdd; 772 } 773 }; 774 775 ALWAYS_INLINE 776 RawShadow *MemToShadow(uptr x) { 777 return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x)); 778 } 779 780 struct MemToMetaImpl { 781 template <typename Mapping> 782 static u32 *Apply(uptr x) { 783 DCHECK(IsAppMemImpl::Apply<Mapping>(x)); 784 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) / 785 kMetaShadowCell * kMetaShadowSize) | 786 Mapping::kMetaShadowBeg); 787 } 788 }; 789 790 ALWAYS_INLINE 791 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); } 792 793 struct ShadowToMemImpl { 794 template <typename Mapping> 795 static uptr Apply(uptr sp) { 796 if (!IsShadowMemImpl::Apply<Mapping>(sp)) 797 return 0; 798 // The shadow mapping is non-linear and we've lost some bits, so we don't 799 // have an easy way to restore the original app address. But the mapping is 800 // a bijection, so we try to restore the address as belonging to 801 // low/mid/high range consecutively and see if shadow->app->shadow mapping 802 // gives us the same address. 803 uptr p = 804 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor; 805 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd && 806 MemToShadowImpl::Apply<Mapping>(p) == sp) 807 return p; 808 if (Mapping::kMidAppMemBeg) { 809 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk); 810 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd && 811 MemToShadowImpl::Apply<Mapping>(p_mid) == sp) 812 return p_mid; 813 } 814 return p | Mapping::kShadowMsk; 815 } 816 }; 817 818 ALWAYS_INLINE 819 uptr ShadowToMem(RawShadow *s) { 820 return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s)); 821 } 822 823 // Compresses addr to kCompressedAddrBits stored in least significant bits. 824 ALWAYS_INLINE uptr CompressAddr(uptr addr) { 825 return addr & ((1ull << kCompressedAddrBits) - 1); 826 } 827 828 struct RestoreAddrImpl { 829 typedef uptr Result; 830 template <typename Mapping> 831 static Result Apply(uptr addr) { 832 // To restore the address we go over all app memory ranges and check if top 833 // 3 bits of the compressed addr match that of the app range. If yes, we 834 // assume that the compressed address come from that range and restore the 835 // missing top bits to match the app range address. 836 const uptr ranges[] = { 837 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg, 838 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd, 839 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd, 840 }; 841 const uptr indicator = 0x0e0000000000ull; 842 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator); 843 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) { 844 uptr beg = ranges[i]; 845 uptr end = ranges[i + 1]; 846 if (beg == end) 847 continue; 848 for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) { 849 if ((addr & indicator) == (p & indicator)) 850 return addr | (p & ~(ind_lsb - 1)); 851 } 852 } 853 Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr); 854 Die(); 855 } 856 }; 857 858 // Restores compressed addr from kCompressedAddrBits to full representation. 859 // This is called only during reporting and is not performance-critical. 860 inline uptr RestoreAddr(uptr addr) { 861 return SelectMapping<RestoreAddrImpl>(addr); 862 } 863 864 void InitializePlatform(); 865 void InitializePlatformEarly(); 866 void CheckAndProtect(); 867 void InitializeShadowMemoryPlatform(); 868 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns); 869 int ExtractResolvFDs(void *state, int *fds, int nfd); 870 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); 871 uptr ExtractLongJmpSp(uptr *env); 872 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size); 873 874 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), 875 void (*cleanup)(void *arg), void *arg); 876 877 void DestroyThreadState(); 878 void PlatformCleanUpThreadState(ThreadState *thr); 879 880 } // namespace __tsan 881 882 #endif // TSAN_PLATFORM_H 883