168d75effSDimitry Andric //===-- msan_poisoning.cpp --------------------------------------*- C++ -*-===//
268d75effSDimitry Andric //
368d75effSDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
468d75effSDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
568d75effSDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
668d75effSDimitry Andric //
768d75effSDimitry Andric //===----------------------------------------------------------------------===//
868d75effSDimitry Andric //
968d75effSDimitry Andric // This file is a part of MemorySanitizer.
1068d75effSDimitry Andric //
1168d75effSDimitry Andric //===----------------------------------------------------------------------===//
1268d75effSDimitry Andric
1368d75effSDimitry Andric #include "msan_poisoning.h"
1468d75effSDimitry Andric
1568d75effSDimitry Andric #include "interception/interception.h"
1668d75effSDimitry Andric #include "msan_origin.h"
17349cc55cSDimitry Andric #include "msan_thread.h"
1868d75effSDimitry Andric #include "sanitizer_common/sanitizer_common.h"
1968d75effSDimitry Andric
2068d75effSDimitry Andric DECLARE_REAL(void *, memset, void *dest, int c, uptr n)
2168d75effSDimitry Andric DECLARE_REAL(void *, memcpy, void *dest, const void *src, uptr n)
2268d75effSDimitry Andric DECLARE_REAL(void *, memmove, void *dest, const void *src, uptr n)
2368d75effSDimitry Andric
2468d75effSDimitry Andric namespace __msan {
2568d75effSDimitry Andric
GetOriginIfPoisoned(uptr addr,uptr size)2668d75effSDimitry Andric u32 GetOriginIfPoisoned(uptr addr, uptr size) {
2768d75effSDimitry Andric unsigned char *s = (unsigned char *)MEM_TO_SHADOW(addr);
2868d75effSDimitry Andric for (uptr i = 0; i < size; ++i)
2968d75effSDimitry Andric if (s[i]) return *(u32 *)SHADOW_TO_ORIGIN(((uptr)s + i) & ~3UL);
3068d75effSDimitry Andric return 0;
3168d75effSDimitry Andric }
3268d75effSDimitry Andric
SetOriginIfPoisoned(uptr addr,uptr src_shadow,uptr size,u32 src_origin)3368d75effSDimitry Andric void SetOriginIfPoisoned(uptr addr, uptr src_shadow, uptr size,
3468d75effSDimitry Andric u32 src_origin) {
3568d75effSDimitry Andric uptr dst_s = MEM_TO_SHADOW(addr);
3668d75effSDimitry Andric uptr src_s = src_shadow;
3768d75effSDimitry Andric uptr src_s_end = src_s + size;
3868d75effSDimitry Andric
3968d75effSDimitry Andric for (; src_s < src_s_end; ++dst_s, ++src_s)
4068d75effSDimitry Andric if (*(u8 *)src_s) *(u32 *)SHADOW_TO_ORIGIN(dst_s & ~3UL) = src_origin;
4168d75effSDimitry Andric }
4268d75effSDimitry Andric
CopyOrigin(const void * dst,const void * src,uptr size,StackTrace * stack)4368d75effSDimitry Andric void CopyOrigin(const void *dst, const void *src, uptr size,
4468d75effSDimitry Andric StackTrace *stack) {
4568d75effSDimitry Andric if (!MEM_IS_APP(dst) || !MEM_IS_APP(src)) return;
4668d75effSDimitry Andric
4768d75effSDimitry Andric uptr d = (uptr)dst;
4868d75effSDimitry Andric uptr beg = d & ~3UL;
4968d75effSDimitry Andric // Copy left unaligned origin if that memory is poisoned.
5068d75effSDimitry Andric if (beg < d) {
51e8d8bef9SDimitry Andric u32 o = GetOriginIfPoisoned((uptr)src, beg + 4 - d);
5268d75effSDimitry Andric if (o) {
5368d75effSDimitry Andric if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack);
5468d75effSDimitry Andric *(u32 *)MEM_TO_ORIGIN(beg) = o;
5568d75effSDimitry Andric }
5668d75effSDimitry Andric beg += 4;
5768d75effSDimitry Andric }
5868d75effSDimitry Andric
5968d75effSDimitry Andric uptr end = (d + size) & ~3UL;
6068d75effSDimitry Andric // If both ends fall into the same 4-byte slot, we are done.
6168d75effSDimitry Andric if (end < beg) return;
6268d75effSDimitry Andric
6368d75effSDimitry Andric // Copy right unaligned origin if that memory is poisoned.
6468d75effSDimitry Andric if (end < d + size) {
6568d75effSDimitry Andric u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end);
6668d75effSDimitry Andric if (o) {
6768d75effSDimitry Andric if (__msan_get_track_origins() > 1) o = ChainOrigin(o, stack);
6868d75effSDimitry Andric *(u32 *)MEM_TO_ORIGIN(end) = o;
6968d75effSDimitry Andric }
7068d75effSDimitry Andric }
7168d75effSDimitry Andric
7268d75effSDimitry Andric if (beg < end) {
7368d75effSDimitry Andric // Align src up.
7468d75effSDimitry Andric uptr s = ((uptr)src + 3) & ~3UL;
7568d75effSDimitry Andric // FIXME: factor out to msan_copy_origin_aligned
7668d75effSDimitry Andric if (__msan_get_track_origins() > 1) {
7768d75effSDimitry Andric u32 *src = (u32 *)MEM_TO_ORIGIN(s);
7868d75effSDimitry Andric u32 *src_s = (u32 *)MEM_TO_SHADOW(s);
7968d75effSDimitry Andric u32 *src_end = (u32 *)MEM_TO_ORIGIN(s + (end - beg));
8068d75effSDimitry Andric u32 *dst = (u32 *)MEM_TO_ORIGIN(beg);
8168d75effSDimitry Andric u32 src_o = 0;
8268d75effSDimitry Andric u32 dst_o = 0;
8368d75effSDimitry Andric for (; src < src_end; ++src, ++src_s, ++dst) {
8468d75effSDimitry Andric if (!*src_s) continue;
8568d75effSDimitry Andric if (*src != src_o) {
8668d75effSDimitry Andric src_o = *src;
8768d75effSDimitry Andric dst_o = ChainOrigin(src_o, stack);
8868d75effSDimitry Andric }
8968d75effSDimitry Andric *dst = dst_o;
9068d75effSDimitry Andric }
9168d75effSDimitry Andric } else {
9268d75effSDimitry Andric REAL(memcpy)((void *)MEM_TO_ORIGIN(beg), (void *)MEM_TO_ORIGIN(s),
9368d75effSDimitry Andric end - beg);
9468d75effSDimitry Andric }
9568d75effSDimitry Andric }
9668d75effSDimitry Andric }
9768d75effSDimitry Andric
ReverseCopyOrigin(const void * dst,const void * src,uptr size,StackTrace * stack)98e8d8bef9SDimitry Andric void ReverseCopyOrigin(const void *dst, const void *src, uptr size,
99e8d8bef9SDimitry Andric StackTrace *stack) {
100e8d8bef9SDimitry Andric if (!MEM_IS_APP(dst) || !MEM_IS_APP(src))
101e8d8bef9SDimitry Andric return;
102e8d8bef9SDimitry Andric
103e8d8bef9SDimitry Andric uptr d = (uptr)dst;
104e8d8bef9SDimitry Andric uptr end = (d + size) & ~3UL;
105e8d8bef9SDimitry Andric
106e8d8bef9SDimitry Andric // Copy right unaligned origin if that memory is poisoned.
107e8d8bef9SDimitry Andric if (end < d + size) {
108e8d8bef9SDimitry Andric u32 o = GetOriginIfPoisoned((uptr)src + (end - d), (d + size) - end);
109e8d8bef9SDimitry Andric if (o) {
110e8d8bef9SDimitry Andric if (__msan_get_track_origins() > 1)
111e8d8bef9SDimitry Andric o = ChainOrigin(o, stack);
112e8d8bef9SDimitry Andric *(u32 *)MEM_TO_ORIGIN(end) = o;
113e8d8bef9SDimitry Andric }
114e8d8bef9SDimitry Andric }
115e8d8bef9SDimitry Andric
116e8d8bef9SDimitry Andric uptr beg = d & ~3UL;
117e8d8bef9SDimitry Andric
118e8d8bef9SDimitry Andric if (beg + 4 < end) {
119e8d8bef9SDimitry Andric // Align src up.
120e8d8bef9SDimitry Andric uptr s = ((uptr)src + 3) & ~3UL;
121e8d8bef9SDimitry Andric if (__msan_get_track_origins() > 1) {
122e8d8bef9SDimitry Andric u32 *src = (u32 *)MEM_TO_ORIGIN(s + end - beg - 4);
123e8d8bef9SDimitry Andric u32 *src_s = (u32 *)MEM_TO_SHADOW(s + end - beg - 4);
124e8d8bef9SDimitry Andric u32 *src_begin = (u32 *)MEM_TO_ORIGIN(s);
125e8d8bef9SDimitry Andric u32 *dst = (u32 *)MEM_TO_ORIGIN(end - 4);
126e8d8bef9SDimitry Andric u32 src_o = 0;
127e8d8bef9SDimitry Andric u32 dst_o = 0;
128e8d8bef9SDimitry Andric for (; src >= src_begin; --src, --src_s, --dst) {
129e8d8bef9SDimitry Andric if (!*src_s)
130e8d8bef9SDimitry Andric continue;
131e8d8bef9SDimitry Andric if (*src != src_o) {
132e8d8bef9SDimitry Andric src_o = *src;
133e8d8bef9SDimitry Andric dst_o = ChainOrigin(src_o, stack);
134e8d8bef9SDimitry Andric }
135e8d8bef9SDimitry Andric *dst = dst_o;
136e8d8bef9SDimitry Andric }
137e8d8bef9SDimitry Andric } else {
138e8d8bef9SDimitry Andric REAL(memmove)
139e8d8bef9SDimitry Andric ((void *)MEM_TO_ORIGIN(beg), (void *)MEM_TO_ORIGIN(s), end - beg - 4);
140e8d8bef9SDimitry Andric }
141e8d8bef9SDimitry Andric }
142e8d8bef9SDimitry Andric
143e8d8bef9SDimitry Andric // Copy left unaligned origin if that memory is poisoned.
144e8d8bef9SDimitry Andric if (beg < d) {
145e8d8bef9SDimitry Andric u32 o = GetOriginIfPoisoned((uptr)src, beg + 4 - d);
146e8d8bef9SDimitry Andric if (o) {
147e8d8bef9SDimitry Andric if (__msan_get_track_origins() > 1)
148e8d8bef9SDimitry Andric o = ChainOrigin(o, stack);
149e8d8bef9SDimitry Andric *(u32 *)MEM_TO_ORIGIN(beg) = o;
150e8d8bef9SDimitry Andric }
151e8d8bef9SDimitry Andric }
152e8d8bef9SDimitry Andric }
153e8d8bef9SDimitry Andric
MoveOrigin(const void * dst,const void * src,uptr size,StackTrace * stack)154e8d8bef9SDimitry Andric void MoveOrigin(const void *dst, const void *src, uptr size,
155e8d8bef9SDimitry Andric StackTrace *stack) {
156e8d8bef9SDimitry Andric // If destination origin range overlaps with source origin range, move
157e8d8bef9SDimitry Andric // origins by coping origins in a reverse order; otherwise, copy origins in
158e8d8bef9SDimitry Andric // a normal order.
159e8d8bef9SDimitry Andric uptr src_aligned_beg = reinterpret_cast<uptr>(src) & ~3UL;
160e8d8bef9SDimitry Andric uptr src_aligned_end = (reinterpret_cast<uptr>(src) + size) & ~3UL;
161e8d8bef9SDimitry Andric uptr dst_aligned_beg = reinterpret_cast<uptr>(dst) & ~3UL;
162e8d8bef9SDimitry Andric if (dst_aligned_beg < src_aligned_end && dst_aligned_beg >= src_aligned_beg)
163e8d8bef9SDimitry Andric return ReverseCopyOrigin(dst, src, size, stack);
164e8d8bef9SDimitry Andric return CopyOrigin(dst, src, size, stack);
165e8d8bef9SDimitry Andric }
166e8d8bef9SDimitry Andric
MoveShadowAndOrigin(const void * dst,const void * src,uptr size,StackTrace * stack)16768d75effSDimitry Andric void MoveShadowAndOrigin(const void *dst, const void *src, uptr size,
16868d75effSDimitry Andric StackTrace *stack) {
16968d75effSDimitry Andric if (!MEM_IS_APP(dst)) return;
17068d75effSDimitry Andric if (!MEM_IS_APP(src)) return;
17168d75effSDimitry Andric if (src == dst) return;
172e8d8bef9SDimitry Andric // MoveOrigin transfers origins by refering to their shadows. So we
173e8d8bef9SDimitry Andric // need to move origins before moving shadows.
174e8d8bef9SDimitry Andric if (__msan_get_track_origins())
175e8d8bef9SDimitry Andric MoveOrigin(dst, src, size, stack);
17668d75effSDimitry Andric REAL(memmove)((void *)MEM_TO_SHADOW((uptr)dst),
17768d75effSDimitry Andric (void *)MEM_TO_SHADOW((uptr)src), size);
17868d75effSDimitry Andric }
17968d75effSDimitry Andric
CopyShadowAndOrigin(const void * dst,const void * src,uptr size,StackTrace * stack)18068d75effSDimitry Andric void CopyShadowAndOrigin(const void *dst, const void *src, uptr size,
18168d75effSDimitry Andric StackTrace *stack) {
18268d75effSDimitry Andric if (!MEM_IS_APP(dst)) return;
18368d75effSDimitry Andric if (!MEM_IS_APP(src)) return;
184e8d8bef9SDimitry Andric // Because origin's range is slightly larger than app range, memcpy may also
185e8d8bef9SDimitry Andric // cause overlapped origin ranges.
18668d75effSDimitry Andric REAL(memcpy)((void *)MEM_TO_SHADOW((uptr)dst),
18768d75effSDimitry Andric (void *)MEM_TO_SHADOW((uptr)src), size);
188e8d8bef9SDimitry Andric if (__msan_get_track_origins())
189e8d8bef9SDimitry Andric MoveOrigin(dst, src, size, stack);
19068d75effSDimitry Andric }
19168d75effSDimitry Andric
CopyMemory(void * dst,const void * src,uptr size,StackTrace * stack)19268d75effSDimitry Andric void CopyMemory(void *dst, const void *src, uptr size, StackTrace *stack) {
19368d75effSDimitry Andric REAL(memcpy)(dst, src, size);
19468d75effSDimitry Andric CopyShadowAndOrigin(dst, src, size, stack);
19568d75effSDimitry Andric }
19668d75effSDimitry Andric
SetShadow(const void * ptr,uptr size,u8 value)19768d75effSDimitry Andric void SetShadow(const void *ptr, uptr size, u8 value) {
19868d75effSDimitry Andric uptr PageSize = GetPageSizeCached();
19968d75effSDimitry Andric uptr shadow_beg = MEM_TO_SHADOW(ptr);
20068d75effSDimitry Andric uptr shadow_end = shadow_beg + size;
20168d75effSDimitry Andric if (value ||
20268d75effSDimitry Andric shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) {
20368d75effSDimitry Andric REAL(memset)((void *)shadow_beg, value, shadow_end - shadow_beg);
20468d75effSDimitry Andric } else {
20568d75effSDimitry Andric uptr page_beg = RoundUpTo(shadow_beg, PageSize);
20668d75effSDimitry Andric uptr page_end = RoundDownTo(shadow_end, PageSize);
20768d75effSDimitry Andric
20868d75effSDimitry Andric if (page_beg >= page_end) {
20968d75effSDimitry Andric REAL(memset)((void *)shadow_beg, 0, shadow_end - shadow_beg);
21068d75effSDimitry Andric } else {
21168d75effSDimitry Andric if (page_beg != shadow_beg) {
21268d75effSDimitry Andric REAL(memset)((void *)shadow_beg, 0, page_beg - shadow_beg);
21368d75effSDimitry Andric }
21468d75effSDimitry Andric if (page_end != shadow_end) {
21568d75effSDimitry Andric REAL(memset)((void *)page_end, 0, shadow_end - page_end);
21668d75effSDimitry Andric }
217fe6060f1SDimitry Andric if (!MmapFixedSuperNoReserve(page_beg, page_end - page_beg))
21868d75effSDimitry Andric Die();
219*06c3fb27SDimitry Andric
220*06c3fb27SDimitry Andric if (__msan_get_track_origins()) {
221*06c3fb27SDimitry Andric // No need to set origin for zero shadow, but we can release pages.
222*06c3fb27SDimitry Andric uptr origin_beg = RoundUpTo(MEM_TO_ORIGIN(ptr), PageSize);
223*06c3fb27SDimitry Andric if (!MmapFixedSuperNoReserve(origin_beg, page_end - page_beg))
224*06c3fb27SDimitry Andric Die();
225*06c3fb27SDimitry Andric }
22668d75effSDimitry Andric }
22768d75effSDimitry Andric }
22868d75effSDimitry Andric }
22968d75effSDimitry Andric
SetOrigin(const void * dst,uptr size,u32 origin)23068d75effSDimitry Andric void SetOrigin(const void *dst, uptr size, u32 origin) {
23168d75effSDimitry Andric // Origin mapping is 4 bytes per 4 bytes of application memory.
23268d75effSDimitry Andric // Here we extend the range such that its left and right bounds are both
23368d75effSDimitry Andric // 4 byte aligned.
23468d75effSDimitry Andric uptr x = MEM_TO_ORIGIN((uptr)dst);
23568d75effSDimitry Andric uptr beg = x & ~3UL; // align down.
23668d75effSDimitry Andric uptr end = (x + size + 3) & ~3UL; // align up.
23768d75effSDimitry Andric u64 origin64 = ((u64)origin << 32) | origin;
23868d75effSDimitry Andric // This is like memset, but the value is 32-bit. We unroll by 2 to write
23968d75effSDimitry Andric // 64 bits at once. May want to unroll further to get 128-bit stores.
24068d75effSDimitry Andric if (beg & 7ULL) {
24168d75effSDimitry Andric *(u32 *)beg = origin;
24268d75effSDimitry Andric beg += 4;
24368d75effSDimitry Andric }
24468d75effSDimitry Andric for (uptr addr = beg; addr < (end & ~7UL); addr += 8) *(u64 *)addr = origin64;
24568d75effSDimitry Andric if (end & 7ULL) *(u32 *)(end - 4) = origin;
24668d75effSDimitry Andric }
24768d75effSDimitry Andric
PoisonMemory(const void * dst,uptr size,StackTrace * stack)24868d75effSDimitry Andric void PoisonMemory(const void *dst, uptr size, StackTrace *stack) {
24968d75effSDimitry Andric SetShadow(dst, size, (u8)-1);
25068d75effSDimitry Andric
25168d75effSDimitry Andric if (__msan_get_track_origins()) {
252349cc55cSDimitry Andric MsanThread *t = GetCurrentThread();
253349cc55cSDimitry Andric if (t && t->InSignalHandler())
254349cc55cSDimitry Andric return;
25568d75effSDimitry Andric Origin o = Origin::CreateHeapOrigin(stack);
25668d75effSDimitry Andric SetOrigin(dst, size, o.raw_id());
25768d75effSDimitry Andric }
25868d75effSDimitry Andric }
25968d75effSDimitry Andric
26068d75effSDimitry Andric } // namespace __msan
261