1 /*
2 Copyright (c) 2005-2020 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15 */
16
17 #ifndef __TBB_scalable_allocator_H
18 #define __TBB_scalable_allocator_H
19 /** @file */
20
21 #include <stddef.h> /* Need ptrdiff_t and size_t from here. */
22 #if !_MSC_VER
23 #include <stdint.h> /* Need intptr_t from here. */
24 #endif
25
26 #if !defined(__cplusplus) && __ICC==1100
27 #pragma warning (push)
28 #pragma warning (disable: 991)
29 #endif
30
31 #ifdef __cplusplus
32 extern "C" {
33 #endif /* __cplusplus */
34
35 #if _MSC_VER >= 1400
36 #define __TBB_EXPORTED_FUNC __cdecl
37 #else
38 #define __TBB_EXPORTED_FUNC
39 #endif
40
41 /** The "malloc" analogue to allocate block of memory of size bytes.
42 * @ingroup memory_allocation */
43 void * __TBB_EXPORTED_FUNC scalable_malloc (size_t size);
44
45 /** The "free" analogue to discard a previously allocated piece of memory.
46 @ingroup memory_allocation */
47 void __TBB_EXPORTED_FUNC scalable_free (void* ptr);
48
49 /** The "realloc" analogue complementing scalable_malloc.
50 @ingroup memory_allocation */
51 void * __TBB_EXPORTED_FUNC scalable_realloc (void* ptr, size_t size);
52
53 /** The "calloc" analogue complementing scalable_malloc.
54 @ingroup memory_allocation */
55 void * __TBB_EXPORTED_FUNC scalable_calloc (size_t nobj, size_t size);
56
57 /** The "posix_memalign" analogue.
58 @ingroup memory_allocation */
59 int __TBB_EXPORTED_FUNC scalable_posix_memalign (void** memptr, size_t alignment, size_t size);
60
61 /** The "_aligned_malloc" analogue.
62 @ingroup memory_allocation */
63 void * __TBB_EXPORTED_FUNC scalable_aligned_malloc (size_t size, size_t alignment);
64
65 /** The "_aligned_realloc" analogue.
66 @ingroup memory_allocation */
67 void * __TBB_EXPORTED_FUNC scalable_aligned_realloc (void* ptr, size_t size, size_t alignment);
68
69 /** The "_aligned_free" analogue.
70 @ingroup memory_allocation */
71 void __TBB_EXPORTED_FUNC scalable_aligned_free (void* ptr);
72
73 /** The analogue of _msize/malloc_size/malloc_usable_size.
74 Returns the usable size of a memory block previously allocated by scalable_*,
75 or 0 (zero) if ptr does not point to such a block.
76 @ingroup memory_allocation */
77 size_t __TBB_EXPORTED_FUNC scalable_msize (void* ptr);
78
79 /* Results for scalable_allocation_* functions */
80 typedef enum {
81 TBBMALLOC_OK,
82 TBBMALLOC_INVALID_PARAM,
83 TBBMALLOC_UNSUPPORTED,
84 TBBMALLOC_NO_MEMORY,
85 TBBMALLOC_NO_EFFECT
86 } ScalableAllocationResult;
87
88 /* Setting TBB_MALLOC_USE_HUGE_PAGES environment variable to 1 enables huge pages.
89 scalable_allocation_mode call has priority over environment variable. */
90 typedef enum {
91 TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */
92 /* deprecated, kept for backward compatibility only */
93 USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES,
94 /* try to limit memory consumption value (Bytes), clean internal buffers
95 if limit is exceeded, but not prevents from requesting memory from OS */
96 TBBMALLOC_SET_SOFT_HEAP_LIMIT,
97 /* Lower bound for the size (Bytes), that is interpreted as huge
98 * and not released during regular cleanup operations. */
99 TBBMALLOC_SET_HUGE_SIZE_THRESHOLD
100 } AllocationModeParam;
101
102 /** Set TBB allocator-specific allocation modes.
103 @ingroup memory_allocation */
104 int __TBB_EXPORTED_FUNC scalable_allocation_mode(int param, intptr_t value);
105
106 typedef enum {
107 /* Clean internal allocator buffers for all threads.
108 Returns TBBMALLOC_NO_EFFECT if no buffers cleaned,
109 TBBMALLOC_OK if some memory released from buffers. */
110 TBBMALLOC_CLEAN_ALL_BUFFERS,
111 /* Clean internal allocator buffer for current thread only.
112 Return values same as for TBBMALLOC_CLEAN_ALL_BUFFERS. */
113 TBBMALLOC_CLEAN_THREAD_BUFFERS
114 } ScalableAllocationCmd;
115
116 /** Call TBB allocator-specific commands.
117 @ingroup memory_allocation */
118 int __TBB_EXPORTED_FUNC scalable_allocation_command(int cmd, void *param);
119
120 #ifdef __cplusplus
121 } /* extern "C" */
122 #endif /* __cplusplus */
123
124 #ifdef __cplusplus
125
126 //! The namespace rml contains components of low-level memory pool interface.
127 namespace rml {
128 class MemoryPool;
129
130 typedef void *(*rawAllocType)(intptr_t pool_id, size_t &bytes);
131 // returns non-zero in case of error
132 typedef int (*rawFreeType)(intptr_t pool_id, void* raw_ptr, size_t raw_bytes);
133
134 /*
135 MemPoolPolicy extension must be compatible with such structure fields layout
136
137 struct MemPoolPolicy {
138 rawAllocType pAlloc;
139 rawFreeType pFree;
140 size_t granularity; // granularity of pAlloc allocations
141 };
142 */
143
144 struct MemPoolPolicy {
145 enum {
146 TBBMALLOC_POOL_VERSION = 1
147 };
148
149 rawAllocType pAlloc;
150 rawFreeType pFree;
151 // granularity of pAlloc allocations. 0 means default used.
152 size_t granularity;
153 int version;
154 // all memory consumed at 1st pAlloc call and never returned,
155 // no more pAlloc calls after 1st
156 unsigned fixedPool : 1,
157 // memory consumed but returned only at pool termination
158 keepAllMemory : 1,
159 reserved : 30;
160
161 MemPoolPolicy(rawAllocType pAlloc_, rawFreeType pFree_,
162 size_t granularity_ = 0, bool fixedPool_ = false,
163 bool keepAllMemory_ = false) :
pAllocMemPoolPolicy164 pAlloc(pAlloc_), pFree(pFree_), granularity(granularity_), version(TBBMALLOC_POOL_VERSION),
165 fixedPool(fixedPool_), keepAllMemory(keepAllMemory_),
166 reserved(0) {}
167 };
168
169 // enums have same values as appropriate enums from ScalableAllocationResult
170 // TODO: use ScalableAllocationResult in pool_create directly
171 enum MemPoolError {
172 // pool created successfully
173 POOL_OK = TBBMALLOC_OK,
174 // invalid policy parameters found
175 INVALID_POLICY = TBBMALLOC_INVALID_PARAM,
176 // requested pool policy is not supported by allocator library
177 UNSUPPORTED_POLICY = TBBMALLOC_UNSUPPORTED,
178 // lack of memory during pool creation
179 NO_MEMORY = TBBMALLOC_NO_MEMORY,
180 // action takes no effect
181 NO_EFFECT = TBBMALLOC_NO_EFFECT
182 };
183
184 MemPoolError pool_create_v1(intptr_t pool_id, const MemPoolPolicy *policy,
185 rml::MemoryPool **pool);
186
187 bool pool_destroy(MemoryPool* memPool);
188 void *pool_malloc(MemoryPool* memPool, size_t size);
189 void *pool_realloc(MemoryPool* memPool, void *object, size_t size);
190 void *pool_aligned_malloc(MemoryPool* mPool, size_t size, size_t alignment);
191 void *pool_aligned_realloc(MemoryPool* mPool, void *ptr, size_t size, size_t alignment);
192 bool pool_reset(MemoryPool* memPool);
193 bool pool_free(MemoryPool *memPool, void *object);
194 MemoryPool *pool_identify(void *object);
195 size_t pool_msize(MemoryPool *memPool, void *object);
196
197 } // namespace rml
198
199 #include <new> /* To use new with the placement argument */
200
201 /* Ensure that including this header does not cause implicit linkage with TBB */
202 #ifndef __TBB_NO_IMPLICIT_LINKAGE
203 #define __TBB_NO_IMPLICIT_LINKAGE 1
204 #include "tbb_stddef.h"
205 #undef __TBB_NO_IMPLICIT_LINKAGE
206 #else
207 #include "tbb_stddef.h"
208 #endif
209
210 #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
211 #include <utility> // std::forward
212 #endif
213
214 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
215 #include <memory_resource>
216 #endif
217
218 namespace tbb {
219
220 #if _MSC_VER && !defined(__INTEL_COMPILER)
221 // Workaround for erroneous "unreferenced parameter" warning in method destroy.
222 #pragma warning (push)
223 #pragma warning (disable: 4100)
224 #endif
225
226 //! @cond INTERNAL
227 namespace internal {
228
229 #if TBB_USE_EXCEPTIONS
230 // forward declaration is for inlining prevention
231 template<typename E> __TBB_NOINLINE( void throw_exception(const E &e) );
232 #endif
233
234 // keep throw in a separate function to prevent code bloat
235 template<typename E>
throw_exception(const E & e)236 void throw_exception(const E &e) {
237 __TBB_THROW(e);
238 }
239
240 } // namespace internal
241 //! @endcond
242
243 //! Meets "allocator" requirements of ISO C++ Standard, Section 20.1.5
244 /** The members are ordered the same way they are in section 20.4.1
245 of the ISO C++ standard.
246 @ingroup memory_allocation */
247 template<typename T>
248 class scalable_allocator {
249 public:
250 typedef typename internal::allocator_type<T>::value_type value_type;
251 typedef value_type* pointer;
252 typedef const value_type* const_pointer;
253 typedef value_type& reference;
254 typedef const value_type& const_reference;
255 typedef size_t size_type;
256 typedef ptrdiff_t difference_type;
257 template<class U> struct rebind {
258 typedef scalable_allocator<U> other;
259 };
260
throw()261 scalable_allocator() throw() {}
throw()262 scalable_allocator( const scalable_allocator& ) throw() {}
scalable_allocator(const scalable_allocator<U> &)263 template<typename U> scalable_allocator(const scalable_allocator<U>&) throw() {}
264
address(reference x)265 pointer address(reference x) const {return &x;}
address(const_reference x)266 const_pointer address(const_reference x) const {return &x;}
267
268 //! Allocate space for n objects.
269 pointer allocate( size_type n, const void* /*hint*/ =0 ) {
270 pointer p = static_cast<pointer>( scalable_malloc( n * sizeof(value_type) ) );
271 if (!p)
272 internal::throw_exception(std::bad_alloc());
273 return p;
274 }
275
276 //! Free previously allocated block of memory
deallocate(pointer p,size_type)277 void deallocate( pointer p, size_type ) {
278 scalable_free( p );
279 }
280
281 //! Largest value for which method allocate might succeed.
max_size()282 size_type max_size() const throw() {
283 size_type absolutemax = static_cast<size_type>(-1) / sizeof (value_type);
284 return (absolutemax > 0 ? absolutemax : 1);
285 }
286 #if __TBB_ALLOCATOR_CONSTRUCT_VARIADIC
287 template<typename U, typename... Args>
construct(U * p,Args &&...args)288 void construct(U *p, Args&&... args)
289 { ::new((void *)p) U(std::forward<Args>(args)...); }
290 #else /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */
291 #if __TBB_CPP11_RVALUE_REF_PRESENT
construct(pointer p,value_type && value)292 void construct( pointer p, value_type&& value ) { ::new((void*)(p)) value_type( std::move( value ) ); }
293 #endif
construct(pointer p,const value_type & value)294 void construct( pointer p, const value_type& value ) {::new((void*)(p)) value_type(value);}
295 #endif /* __TBB_ALLOCATOR_CONSTRUCT_VARIADIC */
destroy(pointer p)296 void destroy( pointer p ) {p->~value_type();}
297 };
298
299 #if _MSC_VER && !defined(__INTEL_COMPILER)
300 #pragma warning (pop)
301 #endif /* warning 4100 is back */
302
303 //! Analogous to std::allocator<void>, as defined in ISO C++ Standard, Section 20.4.1
304 /** @ingroup memory_allocation */
305 template<>
306 class scalable_allocator<void> {
307 public:
308 typedef void* pointer;
309 typedef const void* const_pointer;
310 typedef void value_type;
311 template<class U> struct rebind {
312 typedef scalable_allocator<U> other;
313 };
314 };
315
316 template<typename T, typename U>
317 inline bool operator==( const scalable_allocator<T>&, const scalable_allocator<U>& ) {return true;}
318
319 template<typename T, typename U>
320 inline bool operator!=( const scalable_allocator<T>&, const scalable_allocator<U>& ) {return false;}
321
322 #if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
323
324 namespace internal {
325
326 //! C++17 memory resource implementation for scalable allocator
327 //! ISO C++ Section 23.12.2
328 class scalable_resource_impl : public std::pmr::memory_resource {
329 private:
do_allocate(size_t bytes,size_t alignment)330 void* do_allocate(size_t bytes, size_t alignment) override {
331 void* ptr = scalable_aligned_malloc( bytes, alignment );
332 if (!ptr) {
333 throw_exception(std::bad_alloc());
334 }
335 return ptr;
336 }
337
do_deallocate(void * ptr,size_t,size_t)338 void do_deallocate(void* ptr, size_t /*bytes*/, size_t /*alignment*/) override {
339 scalable_free(ptr);
340 }
341
342 //! Memory allocated by one instance of scalable_resource_impl could be deallocated by any
343 //! other instance of this class
do_is_equal(const std::pmr::memory_resource & other)344 bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
345 return this == &other ||
346 #if __TBB_USE_OPTIONAL_RTTI
347 dynamic_cast<const scalable_resource_impl*>(&other) != NULL;
348 #else
349 false;
350 #endif
351 }
352 };
353
354 } // namespace internal
355
356 //! Global scalable allocator memory resource provider
scalable_memory_resource()357 inline std::pmr::memory_resource* scalable_memory_resource() noexcept {
358 static tbb::internal::scalable_resource_impl scalable_res;
359 return &scalable_res;
360 }
361
362 #endif /* __TBB_CPP17_MEMORY_RESOURCE_PRESENT */
363
364 } // namespace tbb
365
366 #if _MSC_VER
367 #if (__TBB_BUILD || __TBBMALLOC_BUILD) && !defined(__TBBMALLOC_NO_IMPLICIT_LINKAGE)
368 #define __TBBMALLOC_NO_IMPLICIT_LINKAGE 1
369 #endif
370
371 #if !__TBBMALLOC_NO_IMPLICIT_LINKAGE
372 #ifdef _DEBUG
373 #pragma comment(lib, "tbbmalloc_debug.lib")
374 #else
375 #pragma comment(lib, "tbbmalloc.lib")
376 #endif
377 #endif
378
379
380 #endif
381
382 #endif /* __cplusplus */
383
384 #if !defined(__cplusplus) && __ICC==1100
385 #pragma warning (pop)
386 #endif /* ICC 11.0 warning 991 is back */
387
388 #endif /* __TBB_scalable_allocator_H */
389