1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 //
11 // Platform-specific code.
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef TSAN_PLATFORM_H
15 #define TSAN_PLATFORM_H
16 
17 #if !defined(__LP64__) && !defined(_WIN64)
18 # error "Only 64-bit is supported"
19 #endif
20 
21 #include "sanitizer_common/sanitizer_common.h"
22 #include "tsan_defs.h"
23 
24 namespace __tsan {
25 
26 enum {
27   // App memory is not mapped onto shadow memory range.
28   kBrokenMapping = 1 << 0,
29   // Mapping app memory and back does not produce the same address,
30   // this can lead to wrong addresses in reports and potentially
31   // other bad consequences.
32   kBrokenReverseMapping = 1 << 1,
33   // Mapping is non-linear for linear user range.
34   // This is bad and can lead to unpredictable memory corruptions, etc
35   // because range access functions assume linearity.
36   kBrokenLinearity = 1 << 2,
37   // Meta for an app region overlaps with the meta of another app region.
38   // This is determined by recomputing the individual meta regions for
39   // each app region.
40   //
41   // N.B. There is no "kBrokenReverseMetaMapping" constant because there
42   // is no MetaToMem function. However, note that (!kBrokenLinearity
43   // && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
44   kBrokenAliasedMetas = 1 << 3,
45 };
46 
47 /*
48 C/C++ on linux/x86_64 and freebsd/x86_64
49 0000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB)
50 0200 0000 0000 - 1000 0000 0000: -
51 1000 0000 0000 - 3000 0000 0000: shadow (32TB)
52 3000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB)
53 3800 0000 0000 - 5500 0000 0000: -
54 5500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels
55 5a00 0000 0000 - 7200 0000 0000: -
56 7200 0000 0000 - 7300 0000 0000: heap (1TB)
57 7300 0000 0000 - 7a00 0000 0000: -
58 7a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB)
59 
60 C/C++ on netbsd/amd64 can reuse the same mapping:
61  * The address space starts from 0x1000 (option with 0x0) and ends with
62    0x7f7ffffff000.
63  * LoAppMem-kHeapMemEnd can be reused as it is.
64  * No VDSO support.
65  * No MidAppMem region.
66  * No additional HeapMem region.
67  * HiAppMem contains the stack, loader, shared libraries and heap.
68  * Stack on NetBSD/amd64 has prereserved 128MB.
69  * Heap grows downwards (top-down).
70  * ASLR must be disabled per-process or globally.
71 */
72 struct Mapping48AddressSpace {
73   static const uptr kMetaShadowBeg = 0x300000000000ull;
74   static const uptr kMetaShadowEnd = 0x380000000000ull;
75   static const uptr kShadowBeg = 0x100000000000ull;
76   static const uptr kShadowEnd = 0x300000000000ull;
77   static const uptr kHeapMemBeg = 0x720000000000ull;
78   static const uptr kHeapMemEnd = 0x730000000000ull;
79   static const uptr kLoAppMemBeg   = 0x000000001000ull;
80   static const uptr kLoAppMemEnd = 0x020000000000ull;
81   static const uptr kMidAppMemBeg  = 0x550000000000ull;
82   static const uptr kMidAppMemEnd = 0x5a0000000000ull;
83   static const uptr kHiAppMemBeg = 0x7a0000000000ull;
84   static const uptr kHiAppMemEnd   = 0x800000000000ull;
85   static const uptr kShadowMsk = 0x700000000000ull;
86   static const uptr kShadowXor = 0x000000000000ull;
87   static const uptr kShadowAdd = 0x100000000000ull;
88   static const uptr kVdsoBeg       = 0xf000000000000000ull;
89 };
90 
91 /*
92 C/C++ on linux/mips64 (40-bit VMA)
93 0000 0000 00 - 0100 0000 00: -                                           (4 GB)
94 0100 0000 00 - 0200 0000 00: main binary                                 (4 GB)
95 0200 0000 00 - 1200 0000 00: -                                          (64 GB)
96 1200 0000 00 - 2200 0000 00: shadow                                     (64 GB)
97 2200 0000 00 - 4000 0000 00: -                                         (120 GB)
98 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects)  (64 GB)
99 5000 0000 00 - aa00 0000 00: -                                         (360 GB)
100 aa00 0000 00 - ab00 0000 00: main binary (PIE)                           (4 GB)
101 ab00 0000 00 - fe00 0000 00: -                                         (332 GB)
102 fe00 0000 00 - ff00 0000 00: heap                                        (4 GB)
103 ff00 0000 00 - ff80 0000 00: -                                           (2 GB)
104 ff80 0000 00 - ffff ffff ff: modules and main thread stack              (<2 GB)
105 */
106 struct MappingMips64_40 {
107   static const uptr kMetaShadowBeg = 0x4000000000ull;
108   static const uptr kMetaShadowEnd = 0x5000000000ull;
109   static const uptr kShadowBeg = 0x1200000000ull;
110   static const uptr kShadowEnd = 0x2200000000ull;
111   static const uptr kHeapMemBeg    = 0xfe00000000ull;
112   static const uptr kHeapMemEnd    = 0xff00000000ull;
113   static const uptr kLoAppMemBeg   = 0x0100000000ull;
114   static const uptr kLoAppMemEnd   = 0x0200000000ull;
115   static const uptr kMidAppMemBeg  = 0xaa00000000ull;
116   static const uptr kMidAppMemEnd  = 0xab00000000ull;
117   static const uptr kHiAppMemBeg   = 0xff80000000ull;
118   static const uptr kHiAppMemEnd   = 0xffffffffffull;
119   static const uptr kShadowMsk = 0xf800000000ull;
120   static const uptr kShadowXor = 0x0800000000ull;
121   static const uptr kShadowAdd = 0x0000000000ull;
122   static const uptr kVdsoBeg       = 0xfffff00000ull;
123 };
124 
125 /*
126 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
127 0000 0000 00 - 0100 0000 00: -                                    (4 GB)
128 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
129 0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
130 0300 0000 00 - 0400 0000 00: -                                    (4 GB)
131 0400 0000 00 - 0800 0000 00: shadow memory                       (16 GB)
132 0800 0000 00 - 0d00 0000 00: -                                   (20 GB)
133 0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
134 0e00 0000 00 - 1000 0000 00: -
135 */
136 struct MappingAppleAarch64 {
137   static const uptr kLoAppMemBeg   = 0x0100000000ull;
138   static const uptr kLoAppMemEnd   = 0x0200000000ull;
139   static const uptr kHeapMemBeg    = 0x0200000000ull;
140   static const uptr kHeapMemEnd    = 0x0300000000ull;
141   static const uptr kShadowBeg     = 0x0400000000ull;
142   static const uptr kShadowEnd = 0x0800000000ull;
143   static const uptr kMetaShadowBeg = 0x0d00000000ull;
144   static const uptr kMetaShadowEnd = 0x0e00000000ull;
145   static const uptr kHiAppMemBeg   = 0x0fc0000000ull;
146   static const uptr kHiAppMemEnd   = 0x0fc0000000ull;
147   static const uptr kShadowMsk = 0x0ull;
148   static const uptr kShadowXor = 0x0ull;
149   static const uptr kShadowAdd = 0x0200000000ull;
150   static const uptr kVdsoBeg       = 0x7000000000000000ull;
151   static const uptr kMidAppMemBeg = 0;
152   static const uptr kMidAppMemEnd = 0;
153 };
154 
155 /*
156 C/C++ on linux/aarch64 (39-bit VMA)
157 0000 0010 00 - 0500 0000 00: main binary                    (20 GB)
158 0100 0000 00 - 2000 0000 00: -
159 2000 0000 00 - 4000 0000 00: shadow memory                 (128 GB)
160 4000 0000 00 - 4800 0000 00: metainfo                       (32 GB)
161 4800 0000 00 - 5500 0000 00: -
162 5500 0000 00 - 5a00 0000 00: main binary (PIE)              (20 GB)
163 5600 0000 00 - 7c00 0000 00: -
164 7a00 0000 00 - 7d00 0000 00: heap                           (12 GB)
165 7d00 0000 00 - 7fff ffff ff: modules and main thread stack  (12 GB)
166 */
167 struct MappingAarch64_39 {
168   static const uptr kLoAppMemBeg   = 0x0000001000ull;
169   static const uptr kLoAppMemEnd   = 0x0500000000ull;
170   static const uptr kShadowBeg     = 0x2000000000ull;
171   static const uptr kShadowEnd     = 0x4000000000ull;
172   static const uptr kMetaShadowBeg = 0x4000000000ull;
173   static const uptr kMetaShadowEnd = 0x4800000000ull;
174   static const uptr kMidAppMemBeg  = 0x5500000000ull;
175   static const uptr kMidAppMemEnd  = 0x5a00000000ull;
176   static const uptr kHeapMemBeg    = 0x7a00000000ull;
177   static const uptr kHeapMemEnd    = 0x7d00000000ull;
178   static const uptr kHiAppMemBeg   = 0x7d00000000ull;
179   static const uptr kHiAppMemEnd   = 0x7fffffffffull;
180   static const uptr kShadowMsk     = 0x7000000000ull;
181   static const uptr kShadowXor     = 0x1000000000ull;
182   static const uptr kShadowAdd     = 0x0000000000ull;
183   static const uptr kVdsoBeg       = 0x7f00000000ull;
184 };
185 
186 /*
187 C/C++ on linux/aarch64 (42-bit VMA)
188 00000 0010 00 - 02000 0000 00: main binary                   (128 GB)
189 02000 0000 00 - 08000 0000 00: -
190 10000 0000 00 - 20000 0000 00: shadow memory                (1024 GB)
191 20000 0000 00 - 24000 0000 00: metainfo                      (256 GB)
192 24000 0000 00 - 2aa00 0000 00: -
193 2aa00 0000 00 - 2c000 0000 00: main binary (PIE)              (88 GB)
194 2c000 0000 00 - 3c000 0000 00: -
195 3c000 0000 00 - 3f000 0000 00: heap                          (192 GB)
196 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack  (64 GB)
197 */
198 struct MappingAarch64_42 {
199   static const uptr kLoAppMemBeg   = 0x00000001000ull;
200   static const uptr kLoAppMemEnd   = 0x02000000000ull;
201   static const uptr kShadowBeg     = 0x10000000000ull;
202   static const uptr kShadowEnd     = 0x20000000000ull;
203   static const uptr kMetaShadowBeg = 0x20000000000ull;
204   static const uptr kMetaShadowEnd = 0x24000000000ull;
205   static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
206   static const uptr kMidAppMemEnd  = 0x2c000000000ull;
207   static const uptr kHeapMemBeg    = 0x3c000000000ull;
208   static const uptr kHeapMemEnd    = 0x3f000000000ull;
209   static const uptr kHiAppMemBeg   = 0x3f000000000ull;
210   static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
211   static const uptr kShadowMsk     = 0x38000000000ull;
212   static const uptr kShadowXor     = 0x08000000000ull;
213   static const uptr kShadowAdd     = 0x00000000000ull;
214   static const uptr kVdsoBeg       = 0x37f00000000ull;
215 };
216 
217 /*
218 C/C++ on linux/aarch64 (48-bit VMA)
219 0000 0000 1000 - 0a00 0000 0000: main binary                   (10240 GB)
220 0a00 0000 1000 - 1554 0000 0000: -
221 1554 0000 1000 - 5400 0000 0000: shadow memory                 (64176 GB)
222 5400 0000 1000 - 8000 0000 0000: -
223 8000 0000 1000 - 0a00 0000 0000: metainfo                      (32768 GB)
224 a000 0000 1000 - aaaa 0000 0000: -
225 aaaa 0000 1000 - ac00 0000 0000: main binary (PIE)              (1368 GB)
226 ac00 0000 1000 - fc00 0000 0000: -
227 fc00 0000 1000 - ffff ffff ffff: modules and main thread stack  (4096 GB)
228 
229 N.B. the shadow memory region has a strange start address, because it
230 contains the shadows for the mid, high and low app regions (in this
231 unusual order).
232 */
233 struct MappingAarch64_48 {
234   static const uptr kLoAppMemBeg   = 0x0000000001000ull;
235   static const uptr kLoAppMemEnd   = 0x00a0000000000ull;
236   static const uptr kShadowBeg     = 0x0155400000000ull;
237   static const uptr kShadowEnd     = 0x0540000000000ull;
238   static const uptr kMetaShadowBeg = 0x0800000000000ull;
239   static const uptr kMetaShadowEnd = 0x0a00000000000ull;
240   static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
241   static const uptr kMidAppMemEnd  = 0x0ac0000000000ull;
242   static const uptr kHiAppMemBeg   = 0x0fc0000000000ull;
243   static const uptr kHiAppMemEnd   = 0x1000000000000ull;
244   static const uptr kHeapMemBeg    = 0x0fc0000000000ull;
245   static const uptr kHeapMemEnd    = 0x0fc0000000000ull;
246   static const uptr kShadowMsk     = 0x0c00000000000ull;
247   static const uptr kShadowXor     = 0x0200000000000ull;
248   static const uptr kShadowAdd     = 0x0000000000000ull;
249   static const uptr kVdsoBeg       = 0xffff000000000ull;
250 };
251 
252 /* C/C++ on linux/loongarch64 (47-bit VMA)
253 0000 0000 4000 - 0080 0000 0000: main binary
254 0080 0000 0000 - 0100 0000 0000: -
255 0100 0000 0000 - 1000 0000 0000: shadow memory
256 1000 0000 0000 - 3000 0000 0000: -
257 3000 0000 0000 - 3400 0000 0000: metainfo
258 3400 0000 0000 - 5555 0000 0000: -
259 5555 0000 0000 - 5556 0000 0000: main binary (PIE)
260 5556 0000 0000 - 7ffe 0000 0000: -
261 7ffe 0000 0000 - 7fff 0000 0000: heap
262 7fff 0000 0000 - 7fff 8000 0000: -
263 7fff 8000 0000 - 8000 0000 0000: modules and main thread stack
264 */
265 struct MappingLoongArch64_47 {
266   static const uptr kMetaShadowBeg = 0x300000000000ull;
267   static const uptr kMetaShadowEnd = 0x340000000000ull;
268   static const uptr kShadowBeg     = 0x010000000000ull;
269   static const uptr kShadowEnd     = 0x100000000000ull;
270   static const uptr kHeapMemBeg    = 0x7ffe00000000ull;
271   static const uptr kHeapMemEnd    = 0x7fff00000000ull;
272   static const uptr kLoAppMemBeg   = 0x000000004000ull;
273   static const uptr kLoAppMemEnd   = 0x008000000000ull;
274   static const uptr kMidAppMemBeg  = 0x555500000000ull;
275   static const uptr kMidAppMemEnd  = 0x555600000000ull;
276   static const uptr kHiAppMemBeg   = 0x7fff80000000ull;
277   static const uptr kHiAppMemEnd   = 0x800000000000ull;
278   static const uptr kShadowMsk     = 0x780000000000ull;
279   static const uptr kShadowXor     = 0x040000000000ull;
280   static const uptr kShadowAdd     = 0x000000000000ull;
281   static const uptr kVdsoBeg       = 0x7fffffffc000ull;
282 };
283 
284 /*
285 C/C++ on linux/powerpc64 (44-bit VMA)
286 0000 0000 0100 - 0001 0000 0000: main binary
287 0001 0000 0000 - 0001 0000 0000: -
288 0001 0000 0000 - 0b00 0000 0000: shadow
289 0b00 0000 0000 - 0b00 0000 0000: -
290 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
291 0d00 0000 0000 - 0f00 0000 0000: -
292 0f00 0000 0000 - 0f50 0000 0000: heap
293 0f50 0000 0000 - 0f60 0000 0000: -
294 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
295 */
296 struct MappingPPC64_44 {
297   static const uptr kBroken = kBrokenMapping | kBrokenReverseMapping |
298                               kBrokenLinearity | kBrokenAliasedMetas;
299   static const uptr kMetaShadowBeg = 0x0b0000000000ull;
300   static const uptr kMetaShadowEnd = 0x0d0000000000ull;
301   static const uptr kShadowBeg     = 0x000100000000ull;
302   static const uptr kShadowEnd     = 0x0b0000000000ull;
303   static const uptr kLoAppMemBeg   = 0x000000000100ull;
304   static const uptr kLoAppMemEnd   = 0x000100000000ull;
305   static const uptr kHeapMemBeg    = 0x0f0000000000ull;
306   static const uptr kHeapMemEnd    = 0x0f5000000000ull;
307   static const uptr kHiAppMemBeg   = 0x0f6000000000ull;
308   static const uptr kHiAppMemEnd   = 0x100000000000ull; // 44 bits
309   static const uptr kShadowMsk = 0x0f0000000000ull;
310   static const uptr kShadowXor = 0x002100000000ull;
311   static const uptr kShadowAdd = 0x000000000000ull;
312   static const uptr kVdsoBeg       = 0x3c0000000000000ull;
313   static const uptr kMidAppMemBeg = 0;
314   static const uptr kMidAppMemEnd = 0;
315 };
316 
317 /*
318 C/C++ on linux/powerpc64 (46-bit VMA)
319 0000 0000 1000 - 0100 0000 0000: main binary
320 0100 0000 0000 - 0200 0000 0000: -
321 0100 0000 0000 - 0800 0000 0000: shadow
322 0800 0000 0000 - 1000 0000 0000: -
323 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
324 1200 0000 0000 - 3d00 0000 0000: -
325 3d00 0000 0000 - 3e00 0000 0000: heap
326 3e00 0000 0000 - 3e80 0000 0000: -
327 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
328 */
329 struct MappingPPC64_46 {
330   static const uptr kMetaShadowBeg = 0x100000000000ull;
331   static const uptr kMetaShadowEnd = 0x120000000000ull;
332   static const uptr kShadowBeg     = 0x010000000000ull;
333   static const uptr kShadowEnd = 0x080000000000ull;
334   static const uptr kHeapMemBeg    = 0x3d0000000000ull;
335   static const uptr kHeapMemEnd    = 0x3e0000000000ull;
336   static const uptr kLoAppMemBeg   = 0x000000001000ull;
337   static const uptr kLoAppMemEnd   = 0x010000000000ull;
338   static const uptr kHiAppMemBeg   = 0x3e8000000000ull;
339   static const uptr kHiAppMemEnd   = 0x400000000000ull; // 46 bits
340   static const uptr kShadowMsk = 0x3c0000000000ull;
341   static const uptr kShadowXor = 0x020000000000ull;
342   static const uptr kShadowAdd = 0x000000000000ull;
343   static const uptr kVdsoBeg       = 0x7800000000000000ull;
344   static const uptr kMidAppMemBeg = 0;
345   static const uptr kMidAppMemEnd = 0;
346 };
347 
348 /*
349 C/C++ on linux/powerpc64 (47-bit VMA)
350 0000 0000 1000 - 0100 0000 0000: main binary
351 0100 0000 0000 - 0200 0000 0000: -
352 0100 0000 0000 - 0800 0000 0000: shadow
353 0800 0000 0000 - 1000 0000 0000: -
354 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
355 1200 0000 0000 - 7d00 0000 0000: -
356 7d00 0000 0000 - 7e00 0000 0000: heap
357 7e00 0000 0000 - 7e80 0000 0000: -
358 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
359 */
360 struct MappingPPC64_47 {
361   static const uptr kMetaShadowBeg = 0x100000000000ull;
362   static const uptr kMetaShadowEnd = 0x120000000000ull;
363   static const uptr kShadowBeg     = 0x010000000000ull;
364   static const uptr kShadowEnd = 0x080000000000ull;
365   static const uptr kHeapMemBeg    = 0x7d0000000000ull;
366   static const uptr kHeapMemEnd    = 0x7e0000000000ull;
367   static const uptr kLoAppMemBeg   = 0x000000001000ull;
368   static const uptr kLoAppMemEnd   = 0x010000000000ull;
369   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
370   static const uptr kHiAppMemEnd   = 0x800000000000ull; // 47 bits
371   static const uptr kShadowMsk = 0x7c0000000000ull;
372   static const uptr kShadowXor = 0x020000000000ull;
373   static const uptr kShadowAdd = 0x000000000000ull;
374   static const uptr kVdsoBeg       = 0x7800000000000000ull;
375   static const uptr kMidAppMemBeg = 0;
376   static const uptr kMidAppMemEnd = 0;
377 };
378 
379 /*
380 C/C++ on linux/riscv64 (39-bit VMA)
381 0000 0010 00 - 0200 0000 00: main binary                      ( 8 GB)
382 0200 0000 00 - 1000 0000 00: -
383 1000 0000 00 - 4000 0000 00: shadow memory                    (64 GB)
384 4000 0000 00 - 4800 0000 00: metainfo                         (16 GB)
385 4800 0000 00 - 5500 0000 00: -
386 5500 0000 00 - 5a00 0000 00: main binary (PIE)                (~8 GB)
387 5600 0000 00 - 7c00 0000 00: -
388 7d00 0000 00 - 7fff ffff ff: libraries and main thread stack  ( 8 GB)
389 
390 mmap by default allocates from top downwards
391 VDSO sits below loader and above dynamic libraries, within HiApp region.
392 Heap starts after program region whose position depends on pie or non-pie.
393 Disable tracking them since their locations are not fixed.
394 */
395 struct MappingRiscv64_39 {
396   static const uptr kLoAppMemBeg = 0x0000001000ull;
397   static const uptr kLoAppMemEnd = 0x0200000000ull;
398   static const uptr kShadowBeg = 0x1000000000ull;
399   static const uptr kShadowEnd = 0x2000000000ull;
400   static const uptr kMetaShadowBeg = 0x2000000000ull;
401   static const uptr kMetaShadowEnd = 0x2400000000ull;
402   static const uptr kMidAppMemBeg = 0x2aaaaaa000ull;
403   static const uptr kMidAppMemEnd = 0x2c00000000ull;
404   static const uptr kHeapMemBeg = 0x2c00000000ull;
405   static const uptr kHeapMemEnd = 0x2c00000000ull;
406   static const uptr kHiAppMemBeg = 0x3c00000000ull;
407   static const uptr kHiAppMemEnd = 0x3fffffffffull;
408   static const uptr kShadowMsk = 0x3800000000ull;
409   static const uptr kShadowXor = 0x0800000000ull;
410   static const uptr kShadowAdd = 0x0000000000ull;
411   static const uptr kVdsoBeg = 0x4000000000ull;
412 };
413 
414 /*
415 C/C++ on linux/riscv64 (48-bit VMA)
416 0000 0000 1000 - 0400 0000 0000: main binary                      ( 4 TB)
417 0500 0000 0000 - 2000 0000 0000: -
418 2000 0000 0000 - 4000 0000 0000: shadow memory                    (32 TB)
419 4000 0000 0000 - 4800 0000 0000: metainfo                         ( 8 TB)
420 4800 0000 0000 - 5555 5555 5000: -
421 5555 5555 5000 - 5a00 0000 0000: main binary (PIE)                (~5 TB)
422 5a00 0000 0000 - 7a00 0000 0000: -
423 7a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack  ( 6 TB)
424 */
425 struct MappingRiscv64_48 {
426   static const uptr kLoAppMemBeg = 0x000000001000ull;
427   static const uptr kLoAppMemEnd = 0x040000000000ull;
428   static const uptr kShadowBeg = 0x200000000000ull;
429   static const uptr kShadowEnd = 0x400000000000ull;
430   static const uptr kMetaShadowBeg = 0x400000000000ull;
431   static const uptr kMetaShadowEnd = 0x480000000000ull;
432   static const uptr kMidAppMemBeg = 0x555555555000ull;
433   static const uptr kMidAppMemEnd = 0x5a0000000000ull;
434   static const uptr kHeapMemBeg = 0x5a0000000000ull;
435   static const uptr kHeapMemEnd = 0x5a0000000000ull;
436   static const uptr kHiAppMemBeg = 0x7a0000000000ull;
437   static const uptr kHiAppMemEnd = 0x7fffffffffffull;
438   static const uptr kShadowMsk = 0x700000000000ull;
439   static const uptr kShadowXor = 0x100000000000ull;
440   static const uptr kShadowAdd = 0x000000000000ull;
441   static const uptr kVdsoBeg = 0x800000000000ull;
442 };
443 
444 /*
445 C/C++ on linux/s390x
446 While the kernel provides a 64-bit address space, we have to restrict ourselves
447 to 48 bits due to how e.g. SyncVar::GetId() works.
448 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
449 0e00 0000 0000 - 2000 0000 0000: -
450 2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
451 4000 0000 0000 - 9000 0000 0000: -
452 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
453 9800 0000 0000 - be00 0000 0000: -
454 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
455 */
456 struct MappingS390x {
457   static const uptr kMetaShadowBeg = 0x900000000000ull;
458   static const uptr kMetaShadowEnd = 0x980000000000ull;
459   static const uptr kShadowBeg = 0x200000000000ull;
460   static const uptr kShadowEnd = 0x400000000000ull;
461   static const uptr kHeapMemBeg    = 0xbe0000000000ull;
462   static const uptr kHeapMemEnd    = 0xc00000000000ull;
463   static const uptr kLoAppMemBeg   = 0x000000001000ull;
464   static const uptr kLoAppMemEnd   = 0x0e0000000000ull;
465   static const uptr kHiAppMemBeg   = 0xc00000004000ull;
466   static const uptr kHiAppMemEnd   = 0xc00000004000ull;
467   static const uptr kShadowMsk = 0xb00000000000ull;
468   static const uptr kShadowXor = 0x100000000000ull;
469   static const uptr kShadowAdd = 0x000000000000ull;
470   static const uptr kVdsoBeg       = 0xfffffffff000ull;
471   static const uptr kMidAppMemBeg = 0;
472   static const uptr kMidAppMemEnd = 0;
473 };
474 
475 /* Go on linux, darwin and freebsd on x86_64
476 0000 0000 1000 - 0000 1000 0000: executable
477 0000 1000 0000 - 00c0 0000 0000: -
478 00c0 0000 0000 - 00e0 0000 0000: heap
479 00e0 0000 0000 - 2000 0000 0000: -
480 2000 0000 0000 - 21c0 0000 0000: shadow
481 21c0 0000 0000 - 3000 0000 0000: -
482 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
483 4000 0000 0000 - 8000 0000 0000: -
484 */
485 
486 struct MappingGo48 {
487   static const uptr kMetaShadowBeg = 0x300000000000ull;
488   static const uptr kMetaShadowEnd = 0x400000000000ull;
489   static const uptr kShadowBeg     = 0x200000000000ull;
490   static const uptr kShadowEnd = 0x21c000000000ull;
491   static const uptr kLoAppMemBeg = 0x000000001000ull;
492   static const uptr kLoAppMemEnd = 0x00e000000000ull;
493   static const uptr kMidAppMemBeg = 0;
494   static const uptr kMidAppMemEnd = 0;
495   static const uptr kHiAppMemBeg = 0;
496   static const uptr kHiAppMemEnd = 0;
497   static const uptr kHeapMemBeg = 0;
498   static const uptr kHeapMemEnd = 0;
499   static const uptr kVdsoBeg = 0;
500   static const uptr kShadowMsk = 0;
501   static const uptr kShadowXor = 0;
502   static const uptr kShadowAdd = 0x200000000000ull;
503 };
504 
505 /* Go on windows
506 0000 0000 1000 - 0000 1000 0000: executable
507 0000 1000 0000 - 00f8 0000 0000: -
508 00c0 0000 0000 - 00e0 0000 0000: heap
509 00e0 0000 0000 - 0100 0000 0000: -
510 0100 0000 0000 - 0300 0000 0000: shadow
511 0300 0000 0000 - 0700 0000 0000: -
512 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
513 07d0 0000 0000 - 8000 0000 0000: -
514 PIE binaries currently not supported, but it should be theoretically possible.
515 */
516 
517 struct MappingGoWindows {
518   static const uptr kMetaShadowBeg = 0x070000000000ull;
519   static const uptr kMetaShadowEnd = 0x077000000000ull;
520   static const uptr kShadowBeg     = 0x010000000000ull;
521   static const uptr kShadowEnd = 0x030000000000ull;
522   static const uptr kLoAppMemBeg = 0x000000001000ull;
523   static const uptr kLoAppMemEnd = 0x00e000000000ull;
524   static const uptr kMidAppMemBeg = 0;
525   static const uptr kMidAppMemEnd = 0;
526   static const uptr kHiAppMemBeg = 0;
527   static const uptr kHiAppMemEnd = 0;
528   static const uptr kHeapMemBeg = 0;
529   static const uptr kHeapMemEnd = 0;
530   static const uptr kVdsoBeg = 0;
531   static const uptr kShadowMsk = 0;
532   static const uptr kShadowXor = 0;
533   static const uptr kShadowAdd = 0x010000000000ull;
534 };
535 
536 /* Go on linux/powerpc64 (46-bit VMA)
537 0000 0000 1000 - 0000 1000 0000: executable
538 0000 1000 0000 - 00c0 0000 0000: -
539 00c0 0000 0000 - 00e0 0000 0000: heap
540 00e0 0000 0000 - 2000 0000 0000: -
541 2000 0000 0000 - 21c0 0000 0000: shadow
542 21c0 0000 0000 - 2400 0000 0000: -
543 2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
544 2470 0000 0000 - 4000 0000 0000: -
545 */
546 
547 struct MappingGoPPC64_46 {
548   static const uptr kMetaShadowBeg = 0x240000000000ull;
549   static const uptr kMetaShadowEnd = 0x247000000000ull;
550   static const uptr kShadowBeg     = 0x200000000000ull;
551   static const uptr kShadowEnd = 0x21c000000000ull;
552   static const uptr kLoAppMemBeg = 0x000000001000ull;
553   static const uptr kLoAppMemEnd = 0x00e000000000ull;
554   static const uptr kMidAppMemBeg = 0;
555   static const uptr kMidAppMemEnd = 0;
556   static const uptr kHiAppMemBeg = 0;
557   static const uptr kHiAppMemEnd = 0;
558   static const uptr kHeapMemBeg = 0;
559   static const uptr kHeapMemEnd = 0;
560   static const uptr kVdsoBeg = 0;
561   static const uptr kShadowMsk = 0;
562   static const uptr kShadowXor = 0;
563   static const uptr kShadowAdd = 0x200000000000ull;
564 };
565 
566 /* Go on linux/powerpc64 (47-bit VMA)
567 0000 0000 1000 - 0000 1000 0000: executable
568 0000 1000 0000 - 00c0 0000 0000: -
569 00c0 0000 0000 - 00e0 0000 0000: heap
570 00e0 0000 0000 - 2000 0000 0000: -
571 2000 0000 0000 - 2800 0000 0000: shadow
572 2800 0000 0000 - 3000 0000 0000: -
573 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
574 3200 0000 0000 - 8000 0000 0000: -
575 */
576 
577 struct MappingGoPPC64_47 {
578   static const uptr kMetaShadowBeg = 0x300000000000ull;
579   static const uptr kMetaShadowEnd = 0x320000000000ull;
580   static const uptr kShadowBeg     = 0x200000000000ull;
581   static const uptr kShadowEnd = 0x280000000000ull;
582   static const uptr kLoAppMemBeg = 0x000000001000ull;
583   static const uptr kLoAppMemEnd = 0x00e000000000ull;
584   static const uptr kMidAppMemBeg = 0;
585   static const uptr kMidAppMemEnd = 0;
586   static const uptr kHiAppMemBeg = 0;
587   static const uptr kHiAppMemEnd = 0;
588   static const uptr kHeapMemBeg = 0;
589   static const uptr kHeapMemEnd = 0;
590   static const uptr kVdsoBeg = 0;
591   static const uptr kShadowMsk = 0;
592   static const uptr kShadowXor = 0;
593   static const uptr kShadowAdd = 0x200000000000ull;
594 };
595 
596 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
597 0000 0000 1000 - 0000 1000 0000: executable
598 0000 1000 0000 - 00c0 0000 0000: -
599 00c0 0000 0000 - 00e0 0000 0000: heap
600 00e0 0000 0000 - 2000 0000 0000: -
601 2000 0000 0000 - 2800 0000 0000: shadow
602 2800 0000 0000 - 3000 0000 0000: -
603 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
604 3200 0000 0000 - 8000 0000 0000: -
605 */
606 struct MappingGoAarch64 {
607   static const uptr kMetaShadowBeg = 0x300000000000ull;
608   static const uptr kMetaShadowEnd = 0x320000000000ull;
609   static const uptr kShadowBeg     = 0x200000000000ull;
610   static const uptr kShadowEnd = 0x280000000000ull;
611   static const uptr kLoAppMemBeg = 0x000000001000ull;
612   static const uptr kLoAppMemEnd = 0x00e000000000ull;
613   static const uptr kMidAppMemBeg = 0;
614   static const uptr kMidAppMemEnd = 0;
615   static const uptr kHiAppMemBeg = 0;
616   static const uptr kHiAppMemEnd = 0;
617   static const uptr kHeapMemBeg = 0;
618   static const uptr kHeapMemEnd = 0;
619   static const uptr kVdsoBeg = 0;
620   static const uptr kShadowMsk = 0;
621   static const uptr kShadowXor = 0;
622   static const uptr kShadowAdd = 0x200000000000ull;
623 };
624 
625 /* Go on linux/loongarch64 (47-bit VMA)
626 0000 0000 1000 - 0000 1000 0000: executable
627 0000 1000 0000 - 00c0 0000 0000: -
628 00c0 0000 0000 - 00e0 0000 0000: heap
629 00e0 0000 0000 - 2000 0000 0000: -
630 2000 0000 0000 - 2800 0000 0000: shadow
631 2800 0000 0000 - 3000 0000 0000: -
632 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
633 3200 0000 0000 - 8000 0000 0000: -
634 */
635 struct MappingGoLoongArch64_47 {
636   static const uptr kMetaShadowBeg = 0x300000000000ull;
637   static const uptr kMetaShadowEnd = 0x320000000000ull;
638   static const uptr kShadowBeg = 0x200000000000ull;
639   static const uptr kShadowEnd = 0x280000000000ull;
640   static const uptr kLoAppMemBeg = 0x000000001000ull;
641   static const uptr kLoAppMemEnd = 0x00e000000000ull;
642   static const uptr kMidAppMemBeg = 0;
643   static const uptr kMidAppMemEnd = 0;
644   static const uptr kHiAppMemBeg = 0;
645   static const uptr kHiAppMemEnd = 0;
646   static const uptr kHeapMemBeg = 0;
647   static const uptr kHeapMemEnd = 0;
648   static const uptr kVdsoBeg = 0;
649   static const uptr kShadowMsk = 0;
650   static const uptr kShadowXor = 0;
651   static const uptr kShadowAdd = 0x200000000000ull;
652 };
653 
654 /*
655 Go on linux/mips64 (47-bit VMA)
656 0000 0000 1000 - 0000 1000 0000: executable
657 0000 1000 0000 - 00c0 0000 0000: -
658 00c0 0000 0000 - 00e0 0000 0000: heap
659 00e0 0000 0000 - 2000 0000 0000: -
660 2000 0000 0000 - 2800 0000 0000: shadow
661 2800 0000 0000 - 3000 0000 0000: -
662 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
663 3200 0000 0000 - 8000 0000 0000: -
664 */
665 struct MappingGoMips64_47 {
666   static const uptr kMetaShadowBeg = 0x300000000000ull;
667   static const uptr kMetaShadowEnd = 0x320000000000ull;
668   static const uptr kShadowBeg = 0x200000000000ull;
669   static const uptr kShadowEnd = 0x280000000000ull;
670   static const uptr kLoAppMemBeg = 0x000000001000ull;
671   static const uptr kLoAppMemEnd = 0x00e000000000ull;
672   static const uptr kMidAppMemBeg = 0;
673   static const uptr kMidAppMemEnd = 0;
674   static const uptr kHiAppMemBeg = 0;
675   static const uptr kHiAppMemEnd = 0;
676   static const uptr kHeapMemBeg = 0;
677   static const uptr kHeapMemEnd = 0;
678   static const uptr kVdsoBeg = 0;
679   static const uptr kShadowMsk = 0;
680   static const uptr kShadowXor = 0;
681   static const uptr kShadowAdd = 0x200000000000ull;
682 };
683 
684 /*
685 Go on linux/s390x
686 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
687 1000 0000 0000 - 4000 0000 0000: -
688 4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
689 6000 0000 0000 - 9000 0000 0000: -
690 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
691 */
692 struct MappingGoS390x {
693   static const uptr kMetaShadowBeg = 0x900000000000ull;
694   static const uptr kMetaShadowEnd = 0x980000000000ull;
695   static const uptr kShadowBeg     = 0x400000000000ull;
696   static const uptr kShadowEnd = 0x600000000000ull;
697   static const uptr kLoAppMemBeg = 0x000000001000ull;
698   static const uptr kLoAppMemEnd = 0x100000000000ull;
699   static const uptr kMidAppMemBeg = 0;
700   static const uptr kMidAppMemEnd = 0;
701   static const uptr kHiAppMemBeg = 0;
702   static const uptr kHiAppMemEnd = 0;
703   static const uptr kHeapMemBeg = 0;
704   static const uptr kHeapMemEnd = 0;
705   static const uptr kVdsoBeg = 0;
706   static const uptr kShadowMsk = 0;
707   static const uptr kShadowXor = 0;
708   static const uptr kShadowAdd = 0x400000000000ull;
709 };
710 
711 extern uptr vmaSize;
712 
713 template <typename Func, typename Arg>
SelectMapping(Arg arg)714 ALWAYS_INLINE auto SelectMapping(Arg arg) {
715 #if SANITIZER_GO
716 #  if defined(__powerpc64__)
717   switch (vmaSize) {
718     case 46:
719       return Func::template Apply<MappingGoPPC64_46>(arg);
720     case 47:
721       return Func::template Apply<MappingGoPPC64_47>(arg);
722   }
723 #  elif defined(__mips64)
724   return Func::template Apply<MappingGoMips64_47>(arg);
725 #  elif defined(__s390x__)
726   return Func::template Apply<MappingGoS390x>(arg);
727 #  elif defined(__aarch64__)
728   return Func::template Apply<MappingGoAarch64>(arg);
729 #  elif defined(__loongarch_lp64)
730   return Func::template Apply<MappingGoLoongArch64_47>(arg);
731 #  elif SANITIZER_WINDOWS
732   return Func::template Apply<MappingGoWindows>(arg);
733 #  else
734   return Func::template Apply<MappingGo48>(arg);
735 #  endif
736 #else  // SANITIZER_GO
737 #  if SANITIZER_IOS && !SANITIZER_IOSSIM
738   return Func::template Apply<MappingAppleAarch64>(arg);
739 #  elif defined(__x86_64__) || SANITIZER_APPLE
740   return Func::template Apply<Mapping48AddressSpace>(arg);
741 #  elif defined(__aarch64__)
742   switch (vmaSize) {
743     case 39:
744       return Func::template Apply<MappingAarch64_39>(arg);
745     case 42:
746       return Func::template Apply<MappingAarch64_42>(arg);
747     case 48:
748       return Func::template Apply<MappingAarch64_48>(arg);
749   }
750 #  elif SANITIZER_LOONGARCH64
751   return Func::template Apply<MappingLoongArch64_47>(arg);
752 #  elif defined(__powerpc64__)
753   switch (vmaSize) {
754     case 44:
755       return Func::template Apply<MappingPPC64_44>(arg);
756     case 46:
757       return Func::template Apply<MappingPPC64_46>(arg);
758     case 47:
759       return Func::template Apply<MappingPPC64_47>(arg);
760   }
761 #  elif defined(__mips64)
762   return Func::template Apply<MappingMips64_40>(arg);
763 #  elif SANITIZER_RISCV64
764   switch (vmaSize) {
765     case 39:
766       return Func::template Apply<MappingRiscv64_39>(arg);
767     case 48:
768       return Func::template Apply<MappingRiscv64_48>(arg);
769   }
770 #  elif defined(__s390x__)
771   return Func::template Apply<MappingS390x>(arg);
772 #  else
773 #    error "unsupported platform"
774 #  endif
775 #endif
776   Die();
777 }
778 
779 template <typename Func>
ForEachMapping()780 void ForEachMapping() {
781   Func::template Apply<Mapping48AddressSpace>();
782   Func::template Apply<MappingMips64_40>();
783   Func::template Apply<MappingAppleAarch64>();
784   Func::template Apply<MappingAarch64_39>();
785   Func::template Apply<MappingAarch64_42>();
786   Func::template Apply<MappingAarch64_48>();
787   Func::template Apply<MappingLoongArch64_47>();
788   Func::template Apply<MappingPPC64_44>();
789   Func::template Apply<MappingPPC64_46>();
790   Func::template Apply<MappingPPC64_47>();
791   Func::template Apply<MappingRiscv64_39>();
792   Func::template Apply<MappingRiscv64_48>();
793   Func::template Apply<MappingS390x>();
794   Func::template Apply<MappingGo48>();
795   Func::template Apply<MappingGoWindows>();
796   Func::template Apply<MappingGoPPC64_46>();
797   Func::template Apply<MappingGoPPC64_47>();
798   Func::template Apply<MappingGoAarch64>();
799   Func::template Apply<MappingGoLoongArch64_47>();
800   Func::template Apply<MappingGoMips64_47>();
801   Func::template Apply<MappingGoS390x>();
802 }
803 
804 enum MappingType {
805   kLoAppMemBeg,
806   kLoAppMemEnd,
807   kHiAppMemBeg,
808   kHiAppMemEnd,
809   kMidAppMemBeg,
810   kMidAppMemEnd,
811   kHeapMemBeg,
812   kHeapMemEnd,
813   kShadowBeg,
814   kShadowEnd,
815   kMetaShadowBeg,
816   kMetaShadowEnd,
817   kVdsoBeg,
818 };
819 
820 struct MappingField {
821   template <typename Mapping>
ApplyMappingField822   static uptr Apply(MappingType type) {
823     switch (type) {
824       case kLoAppMemBeg:
825         return Mapping::kLoAppMemBeg;
826       case kLoAppMemEnd:
827         return Mapping::kLoAppMemEnd;
828       case kMidAppMemBeg:
829         return Mapping::kMidAppMemBeg;
830       case kMidAppMemEnd:
831         return Mapping::kMidAppMemEnd;
832       case kHiAppMemBeg:
833         return Mapping::kHiAppMemBeg;
834       case kHiAppMemEnd:
835         return Mapping::kHiAppMemEnd;
836       case kHeapMemBeg:
837         return Mapping::kHeapMemBeg;
838       case kHeapMemEnd:
839         return Mapping::kHeapMemEnd;
840       case kVdsoBeg:
841         return Mapping::kVdsoBeg;
842       case kShadowBeg:
843         return Mapping::kShadowBeg;
844       case kShadowEnd:
845         return Mapping::kShadowEnd;
846       case kMetaShadowBeg:
847         return Mapping::kMetaShadowBeg;
848       case kMetaShadowEnd:
849         return Mapping::kMetaShadowEnd;
850     }
851     Die();
852   }
853 };
854 
855 ALWAYS_INLINE
LoAppMemBeg(void)856 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
857 ALWAYS_INLINE
LoAppMemEnd(void)858 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
859 
860 ALWAYS_INLINE
MidAppMemBeg(void)861 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
862 ALWAYS_INLINE
MidAppMemEnd(void)863 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
864 
865 ALWAYS_INLINE
HeapMemBeg(void)866 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
867 ALWAYS_INLINE
HeapMemEnd(void)868 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
869 
870 ALWAYS_INLINE
HiAppMemBeg(void)871 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
872 ALWAYS_INLINE
HiAppMemEnd(void)873 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
874 
875 ALWAYS_INLINE
VdsoBeg(void)876 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
877 
878 ALWAYS_INLINE
ShadowBeg(void)879 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
880 ALWAYS_INLINE
ShadowEnd(void)881 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
882 
883 ALWAYS_INLINE
MetaShadowBeg(void)884 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
885 ALWAYS_INLINE
MetaShadowEnd(void)886 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
887 
888 struct IsAppMemImpl {
889   template <typename Mapping>
ApplyIsAppMemImpl890   static bool Apply(uptr mem) {
891   return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
892          (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
893          (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
894          (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
895   }
896 };
897 
898 ALWAYS_INLINE
IsAppMem(uptr mem)899 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
900 
901 struct IsShadowMemImpl {
902   template <typename Mapping>
ApplyIsShadowMemImpl903   static bool Apply(uptr mem) {
904     return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
905   }
906 };
907 
908 ALWAYS_INLINE
IsShadowMem(RawShadow * p)909 bool IsShadowMem(RawShadow *p) {
910   return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
911 }
912 
913 struct IsMetaMemImpl {
914   template <typename Mapping>
ApplyIsMetaMemImpl915   static bool Apply(uptr mem) {
916     return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
917   }
918 };
919 
920 ALWAYS_INLINE
IsMetaMem(const u32 * p)921 bool IsMetaMem(const u32 *p) {
922   return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
923 }
924 
925 struct MemToShadowImpl {
926   template <typename Mapping>
ApplyMemToShadowImpl927   static uptr Apply(uptr x) {
928     DCHECK(IsAppMemImpl::Apply<Mapping>(x));
929     return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
930             Mapping::kShadowXor) *
931                kShadowMultiplier +
932            Mapping::kShadowAdd;
933   }
934 };
935 
936 ALWAYS_INLINE
MemToShadow(uptr x)937 RawShadow *MemToShadow(uptr x) {
938   return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
939 }
940 
941 struct MemToMetaImpl {
942   template <typename Mapping>
ApplyMemToMetaImpl943   static u32 *Apply(uptr x) {
944     DCHECK(IsAppMemImpl::Apply<Mapping>(x));
945     return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
946                     kMetaShadowCell * kMetaShadowSize) |
947                    Mapping::kMetaShadowBeg);
948   }
949 };
950 
951 ALWAYS_INLINE
MemToMeta(uptr x)952 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
953 
954 struct ShadowToMemImpl {
955   template <typename Mapping>
ApplyShadowToMemImpl956   static uptr Apply(uptr sp) {
957     if (!IsShadowMemImpl::Apply<Mapping>(sp))
958       return 0;
959     // The shadow mapping is non-linear and we've lost some bits, so we don't
960     // have an easy way to restore the original app address. But the mapping is
961     // a bijection, so we try to restore the address as belonging to
962     // low/mid/high range consecutively and see if shadow->app->shadow mapping
963     // gives us the same address.
964     uptr p =
965         ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
966     if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
967         MemToShadowImpl::Apply<Mapping>(p) == sp)
968       return p;
969     if (Mapping::kMidAppMemBeg) {
970       uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
971       if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
972           MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
973         return p_mid;
974     }
975     return p | Mapping::kShadowMsk;
976   }
977 };
978 
979 ALWAYS_INLINE
ShadowToMem(RawShadow * s)980 uptr ShadowToMem(RawShadow *s) {
981   return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
982 }
983 
984 // Compresses addr to kCompressedAddrBits stored in least significant bits.
CompressAddr(uptr addr)985 ALWAYS_INLINE uptr CompressAddr(uptr addr) {
986   return addr & ((1ull << kCompressedAddrBits) - 1);
987 }
988 
989 struct RestoreAddrImpl {
990   typedef uptr Result;
991   template <typename Mapping>
ApplyRestoreAddrImpl992   static Result Apply(uptr addr) {
993     // To restore the address we go over all app memory ranges and check if top
994     // 3 bits of the compressed addr match that of the app range. If yes, we
995     // assume that the compressed address come from that range and restore the
996     // missing top bits to match the app range address.
997     const uptr ranges[] = {
998         Mapping::kLoAppMemBeg,  Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
999         Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
1000         Mapping::kHeapMemBeg,   Mapping::kHeapMemEnd,
1001     };
1002     const uptr indicator = 0x0e0000000000ull;
1003     const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
1004     for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
1005       uptr beg = ranges[i];
1006       uptr end = ranges[i + 1];
1007       if (beg == end)
1008         continue;
1009       for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
1010         if ((addr & indicator) == (p & indicator))
1011           return addr | (p & ~(ind_lsb - 1));
1012       }
1013     }
1014     Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
1015     Die();
1016   }
1017 };
1018 
1019 // Restores compressed addr from kCompressedAddrBits to full representation.
1020 // This is called only during reporting and is not performance-critical.
RestoreAddr(uptr addr)1021 inline uptr RestoreAddr(uptr addr) {
1022   return SelectMapping<RestoreAddrImpl>(addr);
1023 }
1024 
1025 void InitializePlatform();
1026 void InitializePlatformEarly();
1027 bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings);
1028 void InitializeShadowMemoryPlatform();
1029 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
1030 int ExtractResolvFDs(void *state, int *fds, int nfd);
1031 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
1032 uptr ExtractLongJmpSp(uptr *env);
1033 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
1034 
1035 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
1036                                      void (*cleanup)(void *arg), void *arg);
1037 
1038 void DestroyThreadState();
1039 void PlatformCleanUpThreadState(ThreadState *thr);
1040 
1041 }  // namespace __tsan
1042 
1043 #endif  // TSAN_PLATFORM_H
1044