1 /*
2 Copyright (c) 2005-2020 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 #if __linux__ && !__ANDROID__
18 // include <bits/c++config.h> indirectly so that <cstdlib> is not included
19 #include <cstddef>
20 // include <features.h> indirectly so that <stdlib.h> is not included
21 #include <unistd.h>
22 // Working around compiler issue with Anaconda's gcc 7.3 compiler package.
23 // New gcc ported for old libc may provide their inline implementation
24 // of aligned_alloc as required by new C++ standard, this makes it hard to
25 // redefine aligned_alloc here. However, running on systems with new libc
26 // version, it still needs it to be redefined, thus tricking system headers
27 #if defined(__GLIBC_PREREQ) && !__GLIBC_PREREQ(2, 16) && _GLIBCXX_HAVE_ALIGNED_ALLOC
28 // tell <cstdlib> that there is no aligned_alloc
29 #undef _GLIBCXX_HAVE_ALIGNED_ALLOC
30 // trick <stdlib.h> to define another symbol instead
31 #define aligned_alloc __hidden_redefined_aligned_alloc
32 // Fix the state and undefine the trick
33 #include <cstdlib>
34 #undef aligned_alloc
35 #endif // defined(__GLIBC_PREREQ)&&!__GLIBC_PREREQ(2, 16)&&_GLIBCXX_HAVE_ALIGNED_ALLOC
36 #endif // __linux__ && !__ANDROID__
37
38 #include "proxy.h"
39 #include "tbb/tbb_config.h"
40 #include "tbb/tbb_environment.h"
41
42 #if !defined(__EXCEPTIONS) && !defined(_CPPUNWIND) && !defined(__SUNPRO_CC)
43 #if TBB_USE_EXCEPTIONS
44 #error Compilation settings do not support exception handling. Please do not set TBB_USE_EXCEPTIONS macro or set it to 0.
45 #elif !defined(TBB_USE_EXCEPTIONS)
46 #define TBB_USE_EXCEPTIONS 0
47 #endif
48 #elif !defined(TBB_USE_EXCEPTIONS)
49 #define TBB_USE_EXCEPTIONS 1
50 #endif
51
52 #if __TBB_CPP11_PRESENT
53 #define __TBB_THROW_BAD_ALLOC
54 #define __TBB_NO_THROW noexcept
55 #else
56 #define __TBB_THROW_BAD_ALLOC throw(std::bad_alloc)
57 #define __TBB_NO_THROW throw()
58 #endif
59
60 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT
61 /*** internal global operator new implementation (Linux, Windows) ***/
62 #include <new>
63
64 // Synchronization primitives to protect original library pointers and new_handler
65 #include "Synchronize.h"
66
67 #if __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT
68 // Use MallocMutex implementation
69 typedef MallocMutex ProxyMutex;
70 #else
71 // One byte atomic intrinsics are not available,
72 // so use simple pointer based spin mutex
73 class SimpleSpinMutex : tbb::internal::no_copy {
74 intptr_t flag;
75 public:
76 class scoped_lock : tbb::internal::no_copy {
77 SimpleSpinMutex& mutex;
78 public:
scoped_lock(SimpleSpinMutex & m)79 scoped_lock( SimpleSpinMutex& m ) : mutex(m) {
80 while( !(AtomicFetchStore( &(m.flag), 1 ) == 0) );
81 }
~scoped_lock()82 ~scoped_lock() {
83 FencedStore(mutex.flag, 0);
84 }
85 };
86 friend class scoped_lock;
87 };
88 typedef SimpleSpinMutex ProxyMutex;
89 #endif /* __TBB_MSVC_PART_WORD_INTERLOCKED_INTRINSICS_PRESENT */
90
91 // In case there is no std::get_new_handler function
92 // which provides synchronized access to std::new_handler
93 #if !__TBB_CPP11_GET_NEW_HANDLER_PRESENT
94 static ProxyMutex new_lock;
95 #endif
96
InternalOperatorNew(size_t sz)97 static inline void* InternalOperatorNew(size_t sz) {
98 void* res = scalable_malloc(sz);
99 #if TBB_USE_EXCEPTIONS
100 while (!res) {
101 std::new_handler handler;
102 #if __TBB_CPP11_GET_NEW_HANDLER_PRESENT
103 handler = std::get_new_handler();
104 #else
105 {
106 ProxyMutex::scoped_lock lock(new_lock);
107 handler = std::set_new_handler(0);
108 std::set_new_handler(handler);
109 }
110 #endif
111 if (handler) {
112 (*handler)();
113 } else {
114 throw std::bad_alloc();
115 }
116 res = scalable_malloc(sz);
117 }
118 #endif /* TBB_USE_EXCEPTIONS */
119 return res;
120 }
121 /*** end of internal global operator new implementation ***/
122 #endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED || _WIN32 && !__TBB_WIN8UI_SUPPORT
123
124 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED
125
126 #ifndef __THROW
127 #define __THROW
128 #endif
129
130 /*** service functions and variables ***/
131 #include <string.h> // for memset
132 #include <unistd.h> // for sysconf
133
134 static long memoryPageSize;
135
initPageSize()136 static inline void initPageSize()
137 {
138 memoryPageSize = sysconf(_SC_PAGESIZE);
139 }
140
141 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED
142 #include <dlfcn.h>
143 #include <malloc.h> // mallinfo
144
145 /* __TBB_malloc_proxy used as a weak symbol by libtbbmalloc for:
146 1) detection that the proxy library is loaded
147 2) check that dlsym("malloc") found something different from our replacement malloc
148 */
149 // Starting from GCC 9, the -Wmissing-attributes warning was extended for alias below
150 #if __GNUC__ == 9
151 #pragma GCC diagnostic push
152 #pragma GCC diagnostic ignored "-Wmissing-attributes"
153 #endif
154 extern "C" void *__TBB_malloc_proxy(size_t) __attribute__ ((alias ("malloc")));
155 #if __GNUC__ == 9
156 #pragma GCC diagnostic pop
157 #endif
158
159 static void *orig_msize;
160
161 #elif MALLOC_ZONE_OVERLOAD_ENABLED
162
163 #include "proxy_overload_osx.h"
164
165 #endif // MALLOC_ZONE_OVERLOAD_ENABLED
166
167 // Original (i.e., replaced) functions,
168 // they are never changed for MALLOC_ZONE_OVERLOAD_ENABLED.
169 static void *orig_free,
170 *orig_realloc;
171
172 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED
173 #define ZONE_ARG
174 #define PREFIX(name) name
175
176 static void *orig_libc_free,
177 *orig_libc_realloc;
178
179 // We already tried to find ptr to original functions.
180 static intptr_t origFuncSearched;
181
InitOrigPointers()182 inline void InitOrigPointers()
183 {
184 // race is OK here, as different threads found same functions
185 if (!FencedLoad(origFuncSearched)) {
186 orig_free = dlsym(RTLD_NEXT, "free");
187 orig_realloc = dlsym(RTLD_NEXT, "realloc");
188 orig_msize = dlsym(RTLD_NEXT, "malloc_usable_size");
189 orig_libc_free = dlsym(RTLD_NEXT, "__libc_free");
190 orig_libc_realloc = dlsym(RTLD_NEXT, "__libc_realloc");
191
192 FencedStore(origFuncSearched, 1);
193 }
194 }
195
196 /*** replacements for malloc and the family ***/
197 extern "C" {
198 #elif MALLOC_ZONE_OVERLOAD_ENABLED
199
200 // each impl_* function has such 1st argument, it's unused
201 #define ZONE_ARG struct _malloc_zone_t *,
202 #define PREFIX(name) impl_##name
203 // not interested in original functions for zone overload
204 inline void InitOrigPointers() {}
205
206 #endif // MALLOC_UNIXLIKE_OVERLOAD_ENABLED and MALLOC_ZONE_OVERLOAD_ENABLED
207
PREFIX(malloc)208 void *PREFIX(malloc)(ZONE_ARG size_t size) __THROW
209 {
210 return scalable_malloc(size);
211 }
212
PREFIX(calloc)213 void *PREFIX(calloc)(ZONE_ARG size_t num, size_t size) __THROW
214 {
215 return scalable_calloc(num, size);
216 }
217
PREFIX(free)218 void PREFIX(free)(ZONE_ARG void *object) __THROW
219 {
220 InitOrigPointers();
221 __TBB_malloc_safer_free(object, (void (*)(void*))orig_free);
222 }
223
PREFIX(realloc)224 void *PREFIX(realloc)(ZONE_ARG void* ptr, size_t sz) __THROW
225 {
226 InitOrigPointers();
227 return __TBB_malloc_safer_realloc(ptr, sz, orig_realloc);
228 }
229
230 /* The older *NIX interface for aligned allocations;
231 it's formally substituted by posix_memalign and deprecated,
232 so we do not expect it to cause cyclic dependency with C RTL. */
PREFIX(memalign)233 void *PREFIX(memalign)(ZONE_ARG size_t alignment, size_t size) __THROW
234 {
235 return scalable_aligned_malloc(size, alignment);
236 }
237
238 /* valloc allocates memory aligned on a page boundary */
PREFIX(valloc)239 void *PREFIX(valloc)(ZONE_ARG size_t size) __THROW
240 {
241 if (! memoryPageSize) initPageSize();
242
243 return scalable_aligned_malloc(size, memoryPageSize);
244 }
245
246 #undef ZONE_ARG
247 #undef PREFIX
248
249 #if MALLOC_UNIXLIKE_OVERLOAD_ENABLED
250
251 // match prototype from system headers
252 #if __ANDROID__
malloc_usable_size(const void * ptr)253 size_t malloc_usable_size(const void *ptr) __THROW
254 #else
255 size_t malloc_usable_size(void *ptr) __THROW
256 #endif
257 {
258 InitOrigPointers();
259 return __TBB_malloc_safer_msize(const_cast<void*>(ptr), (size_t (*)(void*))orig_msize);
260 }
261
posix_memalign(void ** memptr,size_t alignment,size_t size)262 int posix_memalign(void **memptr, size_t alignment, size_t size) __THROW
263 {
264 return scalable_posix_memalign(memptr, alignment, size);
265 }
266
267 /* pvalloc allocates smallest set of complete pages which can hold
268 the requested number of bytes. Result is aligned on page boundary. */
pvalloc(size_t size)269 void *pvalloc(size_t size) __THROW
270 {
271 if (! memoryPageSize) initPageSize();
272 // align size up to the page size,
273 // pvalloc(0) returns 1 page, see man libmpatrol
274 size = size? ((size-1) | (memoryPageSize-1)) + 1 : memoryPageSize;
275
276 return scalable_aligned_malloc(size, memoryPageSize);
277 }
278
mallopt(int,int)279 int mallopt(int /*param*/, int /*value*/) __THROW
280 {
281 return 1;
282 }
283
mallinfo()284 struct mallinfo mallinfo() __THROW
285 {
286 struct mallinfo m;
287 memset(&m, 0, sizeof(struct mallinfo));
288
289 return m;
290 }
291
292 #if __ANDROID__
293 // Android doesn't have malloc_usable_size, provide it to be compatible
294 // with Linux, in addition overload dlmalloc_usable_size() that presented
295 // under Android.
296 size_t dlmalloc_usable_size(const void *ptr) __attribute__ ((alias ("malloc_usable_size")));
297 #else // __ANDROID__
298 // C11 function, supported starting GLIBC 2.16
299 void *aligned_alloc(size_t alignment, size_t size) __attribute__ ((alias ("memalign")));
300 // Those non-standard functions are exported by GLIBC, and might be used
301 // in conjunction with standard malloc/free, so we must ovberload them.
302 // Bionic doesn't have them. Not removing from the linker scripts,
303 // as absent entry points are ignored by the linker.
304
305 // Starting from GCC 9, the -Wmissing-attributes warning was extended for aliases below
306 #if __GNUC__ == 9
307 #pragma GCC diagnostic push
308 #pragma GCC diagnostic ignored "-Wmissing-attributes"
309 #endif
310 void *__libc_malloc(size_t size) __attribute__ ((alias ("malloc")));
311 void *__libc_calloc(size_t num, size_t size) __attribute__ ((alias ("calloc")));
312 void *__libc_memalign(size_t alignment, size_t size) __attribute__ ((alias ("memalign")));
313 void *__libc_pvalloc(size_t size) __attribute__ ((alias ("pvalloc")));
314 void *__libc_valloc(size_t size) __attribute__ ((alias ("valloc")));
315 #if __GNUC__ == 9
316 #pragma GCC diagnostic pop
317 #endif
318
319 // call original __libc_* to support naive replacement of free via __libc_free etc
__libc_free(void * ptr)320 void __libc_free(void *ptr)
321 {
322 InitOrigPointers();
323 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_libc_free);
324 }
325
__libc_realloc(void * ptr,size_t size)326 void *__libc_realloc(void *ptr, size_t size)
327 {
328 InitOrigPointers();
329 return __TBB_malloc_safer_realloc(ptr, size, orig_libc_realloc);
330 }
331 #endif // !__ANDROID__
332
333 } /* extern "C" */
334
335 /*** replacements for global operators new and delete ***/
336
operator new(size_t sz)337 void* operator new(size_t sz) __TBB_THROW_BAD_ALLOC {
338 return InternalOperatorNew(sz);
339 }
operator new[](size_t sz)340 void* operator new[](size_t sz) __TBB_THROW_BAD_ALLOC {
341 return InternalOperatorNew(sz);
342 }
operator delete(void * ptr)343 void operator delete(void* ptr) __TBB_NO_THROW {
344 InitOrigPointers();
345 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
346 }
operator delete[](void * ptr)347 void operator delete[](void* ptr) __TBB_NO_THROW {
348 InitOrigPointers();
349 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
350 }
operator new(size_t sz,const std::nothrow_t &)351 void* operator new(size_t sz, const std::nothrow_t&) __TBB_NO_THROW {
352 return scalable_malloc(sz);
353 }
operator new[](std::size_t sz,const std::nothrow_t &)354 void* operator new[](std::size_t sz, const std::nothrow_t&) __TBB_NO_THROW {
355 return scalable_malloc(sz);
356 }
operator delete(void * ptr,const std::nothrow_t &)357 void operator delete(void* ptr, const std::nothrow_t&) __TBB_NO_THROW {
358 InitOrigPointers();
359 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
360 }
operator delete[](void * ptr,const std::nothrow_t &)361 void operator delete[](void* ptr, const std::nothrow_t&) __TBB_NO_THROW {
362 InitOrigPointers();
363 __TBB_malloc_safer_free(ptr, (void (*)(void*))orig_free);
364 }
365
366 #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED */
367 #endif /* MALLOC_UNIXLIKE_OVERLOAD_ENABLED || MALLOC_ZONE_OVERLOAD_ENABLED */
368
369 #ifdef _WIN32
370 #include <windows.h>
371
372 #if !__TBB_WIN8UI_SUPPORT
373
374 #include <stdio.h>
375 #include "tbb_function_replacement.h"
376 #include "shared_utils.h"
377
__TBB_malloc_safer_delete(void * ptr)378 void __TBB_malloc_safer_delete( void *ptr)
379 {
380 __TBB_malloc_safer_free( ptr, NULL );
381 }
382
safer_aligned_malloc(size_t size,size_t alignment)383 void* safer_aligned_malloc( size_t size, size_t alignment )
384 {
385 // workaround for "is power of 2 pow N" bug that accepts zeros
386 return scalable_aligned_malloc( size, alignment>sizeof(size_t*)?alignment:sizeof(size_t*) );
387 }
388
389 // we do not support _expand();
safer_expand(void *,size_t)390 void* safer_expand( void *, size_t )
391 {
392 return NULL;
393 }
394
395 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(CRTLIB) \
396 void (*orig_free_##CRTLIB)(void*); \
397 void __TBB_malloc_safer_free_##CRTLIB(void *ptr) \
398 { \
399 __TBB_malloc_safer_free( ptr, orig_free_##CRTLIB ); \
400 } \
401 \
402 void (*orig__aligned_free_##CRTLIB)(void*); \
403 void __TBB_malloc_safer__aligned_free_##CRTLIB(void *ptr) \
404 { \
405 __TBB_malloc_safer_free( ptr, orig__aligned_free_##CRTLIB ); \
406 } \
407 \
408 size_t (*orig__msize_##CRTLIB)(void*); \
409 size_t __TBB_malloc_safer__msize_##CRTLIB(void *ptr) \
410 { \
411 return __TBB_malloc_safer_msize( ptr, orig__msize_##CRTLIB ); \
412 } \
413 \
414 size_t (*orig__aligned_msize_##CRTLIB)(void*, size_t, size_t); \
415 size_t __TBB_malloc_safer__aligned_msize_##CRTLIB( void *ptr, size_t alignment, size_t offset) \
416 { \
417 return __TBB_malloc_safer_aligned_msize( ptr, alignment, offset, orig__aligned_msize_##CRTLIB ); \
418 } \
419 \
420 void* __TBB_malloc_safer_realloc_##CRTLIB( void *ptr, size_t size ) \
421 { \
422 orig_ptrs func_ptrs = {orig_free_##CRTLIB, orig__msize_##CRTLIB}; \
423 return __TBB_malloc_safer_realloc( ptr, size, &func_ptrs ); \
424 } \
425 \
426 void* __TBB_malloc_safer__aligned_realloc_##CRTLIB( void *ptr, size_t size, size_t alignment ) \
427 { \
428 orig_aligned_ptrs func_ptrs = {orig__aligned_free_##CRTLIB, orig__aligned_msize_##CRTLIB}; \
429 return __TBB_malloc_safer_aligned_realloc( ptr, size, alignment, &func_ptrs ); \
430 }
431
432 // Only for ucrtbase: substitution for _o_free
433 void (*orig__o_free)(void*);
__TBB_malloc__o_free(void * ptr)434 void __TBB_malloc__o_free(void *ptr)
435 {
436 __TBB_malloc_safer_free( ptr, orig__o_free );
437 }
438 // Only for ucrtbase: substitution for _free_base
439 void(*orig__free_base)(void*);
__TBB_malloc__free_base(void * ptr)440 void __TBB_malloc__free_base(void *ptr)
441 {
442 __TBB_malloc_safer_free(ptr, orig__free_base);
443 }
444
445 // Size limit is MAX_PATTERN_SIZE (28) byte codes / 56 symbols per line.
446 // * can be used to match any digit in byte codes.
447 // # followed by several * indicate a relative address that needs to be corrected.
448 // Purpose of the pattern is to mark an instruction bound; it should consist of several
449 // full instructions plus one extra byte code. It's not required for the patterns
450 // to be unique (i.e., it's OK to have same pattern for unrelated functions).
451 // TODO: use hot patch prologues if exist
452 const char* known_bytecodes[] = {
453 #if _WIN64
454 // "========================================================" - 56 symbols
455 "4883EC284885C974", // release free()
456 "4883EC284885C975", // release _msize()
457 "4885C974375348", // release free() 8.0.50727.42, 10.0
458 "E907000000CCCC", // release _aligned_msize(), _aligned_free() ucrtbase.dll
459 "C7442410000000008B", // release free() ucrtbase.dll 10.0.14393.33
460 "E90B000000CCCC", // release _msize() ucrtbase.dll 10.0.14393.33
461 "48895C24085748", // release _aligned_msize() ucrtbase.dll 10.0.14393.33
462 "E903000000CCCC", // release _aligned_msize() ucrtbase.dll 10.0.16299.522
463 "48894C24084883EC28BA", // debug prologue
464 "4C894424184889542410", // debug _aligned_msize() 10.0
465 "48894C24084883EC2848", // debug _aligned_free 10.0
466 "488BD1488D0D#*******E9", // _o_free(), ucrtbase.dll
467 #if __TBB_OVERLOAD_OLD_MSVCR
468 "48895C2408574883EC3049", // release _aligned_msize 9.0
469 "4883EC384885C975", // release _msize() 9.0
470 "4C8BC1488B0DA6E4040033", // an old win64 SDK
471 #endif
472 #else // _WIN32
473 // "========================================================" - 56 symbols
474 "8BFF558BEC8B", // multiple
475 "8BFF558BEC83", // release free() & _msize() 10.0.40219.325, _msize() ucrtbase.dll
476 "8BFF558BECFF", // release _aligned_msize ucrtbase.dll
477 "8BFF558BEC51", // release free() & _msize() ucrtbase.dll 10.0.14393.33
478 "558BEC8B450885C074", // release _aligned_free 11.0
479 "558BEC837D08000F", // release _msize() 11.0.51106.1
480 "558BEC837D08007419FF", // release free() 11.0.50727.1
481 "558BEC8B450885C075", // release _aligned_msize() 11.0.50727.1
482 "558BEC6A018B", // debug free() & _msize() 11.0
483 "558BEC8B451050", // debug _aligned_msize() 11.0
484 "558BEC8B450850", // debug _aligned_free 11.0
485 "8BFF558BEC6A", // debug free() & _msize() 10.0.40219.325
486 #if __TBB_OVERLOAD_OLD_MSVCR
487 "6A1868********E8", // release free() 8.0.50727.4053, 9.0
488 "6A1C68********E8", // release _msize() 8.0.50727.4053, 9.0
489 #endif
490 #endif // _WIN64/_WIN32
491 NULL
492 };
493
494 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,function_name,dbgsuffix) \
495 ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \
496 (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, \
497 known_bytecodes, (FUNCPTR*)&orig_##function_name##_##CRT_VER##dbgsuffix );
498
499 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,function_name,dbgsuffix) \
500 ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \
501 (FUNCPTR)__TBB_malloc_safer_##function_name##_##CRT_VER##dbgsuffix, 0, NULL );
502
503 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_REDIRECT(CRT_VER,function_name,dest_func,dbgsuffix) \
504 ReplaceFunctionWithStore( #CRT_VER #dbgsuffix ".dll", #function_name, \
505 (FUNCPTR)__TBB_malloc_safer_##dest_func##_##CRT_VER##dbgsuffix, 0, NULL );
506
507 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,dbgsuffix) \
508 if (BytecodesAreKnown(#CRT_VER #dbgsuffix ".dll")) { \
509 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,free,dbgsuffix) \
510 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_msize,dbgsuffix) \
511 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,realloc,dbgsuffix) \
512 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_aligned_free,dbgsuffix) \
513 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY(CRT_VER,_aligned_msize,dbgsuffix) \
514 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_ENTRY_NO_FALLBACK(CRT_VER,_aligned_realloc,dbgsuffix) \
515 } else \
516 SkipReplacement(#CRT_VER #dbgsuffix ".dll");
517
518 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(CRT_VER) __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,)
519 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_DEBUG(CRT_VER) __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_IMPL(CRT_VER,d)
520
521 #define __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(CRT_VER) \
522 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(CRT_VER) \
523 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_DEBUG(CRT_VER)
524
525 #if __TBB_OVERLOAD_OLD_MSVCR
526 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr70d);
527 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr70);
528 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr71d);
529 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr71);
530 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr80d);
531 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr80);
532 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr90d);
533 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr90);
534 #endif
535 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr100d);
536 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr100);
537 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr110d);
538 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr110);
539 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120d);
540 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(msvcr120);
541 __TBB_ORIG_ALLOCATOR_REPLACEMENT_WRAPPER(ucrtbase);
542
543 /*** replacements for global operators new and delete ***/
544
545 #if _MSC_VER && !defined(__INTEL_COMPILER)
546 #pragma warning( push )
547 #pragma warning( disable : 4290 )
548 #endif
549
550 /*** operator new overloads internals (Linux, Windows) ***/
551
operator_new(size_t sz)552 void* operator_new(size_t sz) __TBB_THROW_BAD_ALLOC {
553 return InternalOperatorNew(sz);
554 }
operator_new_arr(size_t sz)555 void* operator_new_arr(size_t sz) __TBB_THROW_BAD_ALLOC {
556 return InternalOperatorNew(sz);
557 }
operator_delete(void * ptr)558 void operator_delete(void* ptr) __TBB_NO_THROW {
559 __TBB_malloc_safer_delete(ptr);
560 }
561 #if _MSC_VER && !defined(__INTEL_COMPILER)
562 #pragma warning( pop )
563 #endif
564
operator_delete_arr(void * ptr)565 void operator_delete_arr(void* ptr) __TBB_NO_THROW {
566 __TBB_malloc_safer_delete(ptr);
567 }
operator_new_t(size_t sz,const std::nothrow_t &)568 void* operator_new_t(size_t sz, const std::nothrow_t&) __TBB_NO_THROW {
569 return scalable_malloc(sz);
570 }
operator_new_arr_t(std::size_t sz,const std::nothrow_t &)571 void* operator_new_arr_t(std::size_t sz, const std::nothrow_t&) __TBB_NO_THROW {
572 return scalable_malloc(sz);
573 }
operator_delete_t(void * ptr,const std::nothrow_t &)574 void operator_delete_t(void* ptr, const std::nothrow_t&) __TBB_NO_THROW {
575 __TBB_malloc_safer_delete(ptr);
576 }
operator_delete_arr_t(void * ptr,const std::nothrow_t &)577 void operator_delete_arr_t(void* ptr, const std::nothrow_t&) __TBB_NO_THROW {
578 __TBB_malloc_safer_delete(ptr);
579 }
580
581 struct Module {
582 const char *name;
583 bool doFuncReplacement; // do replacement in the DLL
584 };
585
586 Module modules_to_replace[] = {
587 {"msvcr100d.dll", true},
588 {"msvcr100.dll", true},
589 {"msvcr110d.dll", true},
590 {"msvcr110.dll", true},
591 {"msvcr120d.dll", true},
592 {"msvcr120.dll", true},
593 {"ucrtbase.dll", true},
594 // "ucrtbased.dll" is not supported because of problems with _dbg functions
595 #if __TBB_OVERLOAD_OLD_MSVCR
596 {"msvcr90d.dll", true},
597 {"msvcr90.dll", true},
598 {"msvcr80d.dll", true},
599 {"msvcr80.dll", true},
600 {"msvcr70d.dll", true},
601 {"msvcr70.dll", true},
602 {"msvcr71d.dll", true},
603 {"msvcr71.dll", true},
604 #endif
605 #if __TBB_TODO
606 // TODO: Try enabling replacement for non-versioned system binaries below
607 {"msvcrtd.dll", true},
608 {"msvcrt.dll", true},
609 #endif
610 };
611
612 /*
613 We need to replace following functions:
614 malloc
615 calloc
616 _aligned_malloc
617 _expand (by dummy implementation)
618 ??2@YAPAXI@Z operator new (ia32)
619 ??_U@YAPAXI@Z void * operator new[] (size_t size) (ia32)
620 ??3@YAXPAX@Z operator delete (ia32)
621 ??_V@YAXPAX@Z operator delete[] (ia32)
622 ??2@YAPEAX_K@Z void * operator new(unsigned __int64) (intel64)
623 ??_V@YAXPEAX@Z void * operator new[](unsigned __int64) (intel64)
624 ??3@YAXPEAX@Z operator delete (intel64)
625 ??_V@YAXPEAX@Z operator delete[] (intel64)
626 ??2@YAPAXIABUnothrow_t@std@@@Z void * operator new (size_t sz, const std::nothrow_t&) throw() (optional)
627 ??_U@YAPAXIABUnothrow_t@std@@@Z void * operator new[] (size_t sz, const std::nothrow_t&) throw() (optional)
628
629 and these functions have runtime-specific replacement:
630 realloc
631 free
632 _msize
633 _aligned_realloc
634 _aligned_free
635 _aligned_msize
636 */
637
638 typedef struct FRData_t {
639 //char *_module;
640 const char *_func;
641 FUNCPTR _fptr;
642 FRR_ON_ERROR _on_error;
643 } FRDATA;
644
645 FRDATA c_routines_to_replace[] = {
646 { "malloc", (FUNCPTR)scalable_malloc, FRR_FAIL },
647 { "calloc", (FUNCPTR)scalable_calloc, FRR_FAIL },
648 { "_aligned_malloc", (FUNCPTR)safer_aligned_malloc, FRR_FAIL },
649 { "_expand", (FUNCPTR)safer_expand, FRR_IGNORE },
650 };
651
652 FRDATA cxx_routines_to_replace[] = {
653 #if _WIN64
654 { "??2@YAPEAX_K@Z", (FUNCPTR)operator_new, FRR_FAIL },
655 { "??_U@YAPEAX_K@Z", (FUNCPTR)operator_new_arr, FRR_FAIL },
656 { "??3@YAXPEAX@Z", (FUNCPTR)operator_delete, FRR_FAIL },
657 { "??_V@YAXPEAX@Z", (FUNCPTR)operator_delete_arr, FRR_FAIL },
658 #else
659 { "??2@YAPAXI@Z", (FUNCPTR)operator_new, FRR_FAIL },
660 { "??_U@YAPAXI@Z", (FUNCPTR)operator_new_arr, FRR_FAIL },
661 { "??3@YAXPAX@Z", (FUNCPTR)operator_delete, FRR_FAIL },
662 { "??_V@YAXPAX@Z", (FUNCPTR)operator_delete_arr, FRR_FAIL },
663 #endif
664 { "??2@YAPAXIABUnothrow_t@std@@@Z", (FUNCPTR)operator_new_t, FRR_IGNORE },
665 { "??_U@YAPAXIABUnothrow_t@std@@@Z", (FUNCPTR)operator_new_arr_t, FRR_IGNORE }
666 };
667
668 #ifndef UNICODE
669 typedef char unicode_char_t;
670 #define WCHAR_SPEC "%s"
671 #else
672 typedef wchar_t unicode_char_t;
673 #define WCHAR_SPEC "%ls"
674 #endif
675
676 // Check that we recognize bytecodes that should be replaced by trampolines.
677 // If some functions have unknown prologue patterns, replacement should not be done.
BytecodesAreKnown(const unicode_char_t * dllName)678 bool BytecodesAreKnown(const unicode_char_t *dllName)
679 {
680 const char *funcName[] = {"free", "_msize", "_aligned_free", "_aligned_msize", 0};
681 HMODULE module = GetModuleHandle(dllName);
682
683 if (!module)
684 return false;
685 for (int i=0; funcName[i]; i++)
686 if (! IsPrologueKnown(dllName, funcName[i], known_bytecodes, module)) {
687 fprintf(stderr, "TBBmalloc: skip allocation functions replacement in " WCHAR_SPEC
688 ": unknown prologue for function " WCHAR_SPEC "\n", dllName, funcName[i]);
689 return false;
690 }
691 return true;
692 }
693
SkipReplacement(const unicode_char_t * dllName)694 void SkipReplacement(const unicode_char_t *dllName)
695 {
696 #ifndef UNICODE
697 const char *dllStr = dllName;
698 #else
699 const size_t sz = 128; // all DLL name must fit
700
701 char buffer[sz];
702 size_t real_sz;
703 char *dllStr = buffer;
704
705 errno_t ret = wcstombs_s(&real_sz, dllStr, sz, dllName, sz-1);
706 __TBB_ASSERT(!ret, "Dll name conversion failed");
707 #endif
708
709 for (size_t i=0; i<arrayLength(modules_to_replace); i++)
710 if (!strcmp(modules_to_replace[i].name, dllStr)) {
711 modules_to_replace[i].doFuncReplacement = false;
712 break;
713 }
714 }
715
ReplaceFunctionWithStore(const unicode_char_t * dllName,const char * funcName,FUNCPTR newFunc,const char ** opcodes,FUNCPTR * origFunc,FRR_ON_ERROR on_error=FRR_FAIL)716 void ReplaceFunctionWithStore( const unicode_char_t *dllName, const char *funcName, FUNCPTR newFunc, const char ** opcodes, FUNCPTR* origFunc, FRR_ON_ERROR on_error = FRR_FAIL )
717 {
718 FRR_TYPE res = ReplaceFunction( dllName, funcName, newFunc, opcodes, origFunc );
719
720 if (res == FRR_OK || res == FRR_NODLL || (res == FRR_NOFUNC && on_error == FRR_IGNORE))
721 return;
722
723 fprintf(stderr, "Failed to %s function %s in module %s\n",
724 res==FRR_NOFUNC? "find" : "replace", funcName, dllName);
725
726 // Unable to replace a required function
727 // Aborting because incomplete replacement of memory management functions
728 // may leave the program in an invalid state
729 abort();
730 }
731
doMallocReplacement()732 void doMallocReplacement()
733 {
734 // Replace functions and keep backup of original code (separate for each runtime)
735 #if __TBB_OVERLOAD_OLD_MSVCR
736 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr70)
737 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr71)
738 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr80)
739 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr90)
740 #endif
741 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr100)
742 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr110)
743 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL(msvcr120)
744 __TBB_ORIG_ALLOCATOR_REPLACEMENT_CALL_RELEASE(ucrtbase)
745
746 // Replace functions without storing original code
747 for (size_t j = 0; j < arrayLength(modules_to_replace); j++) {
748 if (!modules_to_replace[j].doFuncReplacement)
749 continue;
750 for (size_t i = 0; i < arrayLength(c_routines_to_replace); i++)
751 {
752 ReplaceFunctionWithStore( modules_to_replace[j].name, c_routines_to_replace[i]._func, c_routines_to_replace[i]._fptr, NULL, NULL, c_routines_to_replace[i]._on_error );
753 }
754 if ( strcmp(modules_to_replace[j].name, "ucrtbase.dll") == 0 ) {
755 HMODULE ucrtbase_handle = GetModuleHandle("ucrtbase.dll");
756 if (!ucrtbase_handle)
757 continue;
758 // If _o_free function is present and patchable, redirect it to tbbmalloc as well
759 // This prevents issues with other _o_* functions which might allocate memory with malloc
760 if ( IsPrologueKnown("ucrtbase.dll", "_o_free", known_bytecodes, ucrtbase_handle)) {
761 ReplaceFunctionWithStore( "ucrtbase.dll", "_o_free", (FUNCPTR)__TBB_malloc__o_free, known_bytecodes, (FUNCPTR*)&orig__o_free, FRR_FAIL );
762 }
763 // Similarly for _free_base
764 if (IsPrologueKnown("ucrtbase.dll", "_free_base", known_bytecodes, ucrtbase_handle)) {
765 ReplaceFunctionWithStore("ucrtbase.dll", "_free_base", (FUNCPTR)__TBB_malloc__free_base, known_bytecodes, (FUNCPTR*)&orig__free_base, FRR_FAIL);
766 }
767 // ucrtbase.dll does not export operator new/delete, so skip the rest of the loop.
768 continue;
769 }
770
771 for (size_t i = 0; i < arrayLength(cxx_routines_to_replace); i++)
772 {
773 #if !_WIN64
774 // in Microsoft* Visual Studio* 2012 and 2013 32-bit operator delete consists of 2 bytes only: short jump to free(ptr);
775 // replacement should be skipped for this particular case.
776 if ( ((strcmp(modules_to_replace[j].name, "msvcr110.dll") == 0) || (strcmp(modules_to_replace[j].name, "msvcr120.dll") == 0)) && (strcmp(cxx_routines_to_replace[i]._func, "??3@YAXPAX@Z") == 0) ) continue;
777 // in Microsoft* Visual Studio* 2013 32-bit operator delete[] consists of 2 bytes only: short jump to free(ptr);
778 // replacement should be skipped for this particular case.
779 if ( (strcmp(modules_to_replace[j].name, "msvcr120.dll") == 0) && (strcmp(cxx_routines_to_replace[i]._func, "??_V@YAXPAX@Z") == 0) ) continue;
780 #endif
781 ReplaceFunctionWithStore( modules_to_replace[j].name, cxx_routines_to_replace[i]._func, cxx_routines_to_replace[i]._fptr, NULL, NULL, cxx_routines_to_replace[i]._on_error );
782 }
783 }
784 }
785
786 #endif // !__TBB_WIN8UI_SUPPORT
787
DllMain(HINSTANCE hInst,DWORD callReason,LPVOID reserved)788 extern "C" BOOL WINAPI DllMain( HINSTANCE hInst, DWORD callReason, LPVOID reserved )
789 {
790
791 if ( callReason==DLL_PROCESS_ATTACH && reserved && hInst ) {
792 #if !__TBB_WIN8UI_SUPPORT
793 if (!tbb::internal::GetBoolEnvironmentVariable("TBB_MALLOC_DISABLE_REPLACEMENT"))
794 {
795 doMallocReplacement();
796 }
797 #endif // !__TBB_WIN8UI_SUPPORT
798 }
799
800 return TRUE;
801 }
802
803 // Just to make the linker happy and link the DLL to the application
__TBB_malloc_proxy()804 extern "C" __declspec(dllexport) void __TBB_malloc_proxy()
805 {
806
807 }
808
809 #endif //_WIN32
810