1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // This file defines an Arena allocator for better allocation performance.
32
33 #ifndef GOOGLE_PROTOBUF_ARENA_H__
34 #define GOOGLE_PROTOBUF_ARENA_H__
35
36
37 #include <limits>
38 #include <type_traits>
39 #include <utility>
40 #ifdef max
41 #undef max // Visual Studio defines this macro
42 #endif
43 #if defined(_MSC_VER) && !defined(_LIBCPP_STD_VER) && !_HAS_EXCEPTIONS
44 // Work around bugs in MSVC <typeinfo> header when _HAS_EXCEPTIONS=0.
45 #include <exception>
46 #include <typeinfo>
47 namespace std {
48 using type_info = ::type_info;
49 }
50 #else
51 #include <typeinfo>
52 #endif
53
54 #include <type_traits>
55 #include <google/protobuf/arena_impl.h>
56 #include <google/protobuf/port.h>
57
58 #include <google/protobuf/port_def.inc>
59
60 #ifdef SWIG
61 #error "You cannot SWIG proto headers"
62 #endif
63
64 namespace google {
65 namespace protobuf {
66
67 struct ArenaOptions; // defined below
68
69 } // namespace protobuf
70 } // namespace google
71
72 namespace google {
73 namespace protobuf {
74
75 class Arena; // defined below
76 class Message; // defined in message.h
77 class MessageLite;
78 template <typename Key, typename T>
79 class Map;
80
81 namespace arena_metrics {
82
83 void EnableArenaMetrics(ArenaOptions* options);
84
85 } // namespace arena_metrics
86
87 namespace TestUtil {
88 class ReflectionTester; // defined in test_util.h
89 } // namespace TestUtil
90
91 namespace internal {
92
93 struct ArenaStringPtr; // defined in arenastring.h
94 class LazyField; // defined in lazy_field.h
95 class EpsCopyInputStream; // defined in parse_context.h
96
97 template <typename Type>
98 class GenericTypeHandler; // defined in repeated_field.h
99
100 inline PROTOBUF_ALWAYS_INLINE
AlignTo(void * ptr,size_t align)101 void* AlignTo(void* ptr, size_t align) {
102 return reinterpret_cast<void*>(
103 (reinterpret_cast<uintptr_t>(ptr) + align - 1) & (~align + 1));
104 }
105
106 // Templated cleanup methods.
107 template <typename T>
arena_destruct_object(void * object)108 void arena_destruct_object(void* object) {
109 reinterpret_cast<T*>(object)->~T();
110 }
111
112 template <bool destructor_skippable, typename T>
113 struct ObjectDestructor {
114 constexpr static void (*destructor)(void*) = &arena_destruct_object<T>;
115 };
116
117 template <typename T>
118 struct ObjectDestructor<true, T> {
119 constexpr static void (*destructor)(void*) = nullptr;
120 };
121
122 template <typename T>
123 void arena_delete_object(void* object) {
124 delete reinterpret_cast<T*>(object);
125 }
126 } // namespace internal
127
128 // ArenaOptions provides optional additional parameters to arena construction
129 // that control its block-allocation behavior.
130 struct ArenaOptions {
131 // This defines the size of the first block requested from the system malloc.
132 // Subsequent block sizes will increase in a geometric series up to a maximum.
133 size_t start_block_size;
134
135 // This defines the maximum block size requested from system malloc (unless an
136 // individual arena allocation request occurs with a size larger than this
137 // maximum). Requested block sizes increase up to this value, then remain
138 // here.
139 size_t max_block_size;
140
141 // An initial block of memory for the arena to use, or NULL for none. If
142 // provided, the block must live at least as long as the arena itself. The
143 // creator of the Arena retains ownership of the block after the Arena is
144 // destroyed.
145 char* initial_block;
146
147 // The size of the initial block, if provided.
148 size_t initial_block_size;
149
150 // A function pointer to an alloc method that returns memory blocks of size
151 // requested. By default, it contains a ptr to the malloc function.
152 //
153 // NOTE: block_alloc and dealloc functions are expected to behave like
154 // malloc and free, including Asan poisoning.
155 void* (*block_alloc)(size_t);
156 // A function pointer to a dealloc method that takes ownership of the blocks
157 // from the arena. By default, it contains a ptr to a wrapper function that
158 // calls free.
159 void (*block_dealloc)(void*, size_t);
160
161 ArenaOptions()
162 : start_block_size(internal::AllocationPolicy::kDefaultStartBlockSize),
163 max_block_size(internal::AllocationPolicy::kDefaultMaxBlockSize),
164 initial_block(NULL),
165 initial_block_size(0),
166 block_alloc(nullptr),
167 block_dealloc(nullptr),
168 make_metrics_collector(nullptr) {}
169
170 private:
171 // If make_metrics_collector is not nullptr, it will be called at Arena init
172 // time. It may return a pointer to a collector instance that will be notified
173 // of interesting events related to the arena.
174 internal::ArenaMetricsCollector* (*make_metrics_collector)();
175
176 internal::ArenaMetricsCollector* MetricsCollector() const {
177 return make_metrics_collector ? (*make_metrics_collector)() : nullptr;
178 }
179
180 internal::AllocationPolicy AllocationPolicy() const {
181 internal::AllocationPolicy res;
182 res.start_block_size = start_block_size;
183 res.max_block_size = max_block_size;
184 res.block_alloc = block_alloc;
185 res.block_dealloc = block_dealloc;
186 res.metrics_collector = MetricsCollector();
187 return res;
188 }
189
190 friend void arena_metrics::EnableArenaMetrics(ArenaOptions*);
191
192 friend class Arena;
193 friend class ArenaOptionsTestFriend;
194 };
195
196 // Support for non-RTTI environments. (The metrics hooks API uses type
197 // information.)
198 #if PROTOBUF_RTTI
199 #define RTTI_TYPE_ID(type) (&typeid(type))
200 #else
201 #define RTTI_TYPE_ID(type) (NULL)
202 #endif
203
204 // Arena allocator. Arena allocation replaces ordinary (heap-based) allocation
205 // with new/delete, and improves performance by aggregating allocations into
206 // larger blocks and freeing allocations all at once. Protocol messages are
207 // allocated on an arena by using Arena::CreateMessage<T>(Arena*), below, and
208 // are automatically freed when the arena is destroyed.
209 //
210 // This is a thread-safe implementation: multiple threads may allocate from the
211 // arena concurrently. Destruction is not thread-safe and the destructing
212 // thread must synchronize with users of the arena first.
213 //
214 // An arena provides two allocation interfaces: CreateMessage<T>, which works
215 // for arena-enabled proto2 message types as well as other types that satisfy
216 // the appropriate protocol (described below), and Create<T>, which works for
217 // any arbitrary type T. CreateMessage<T> is better when the type T supports it,
218 // because this interface (i) passes the arena pointer to the created object so
219 // that its sub-objects and internal allocations can use the arena too, and (ii)
220 // elides the object's destructor call when possible. Create<T> does not place
221 // any special requirements on the type T, and will invoke the object's
222 // destructor when the arena is destroyed.
223 //
224 // The arena message allocation protocol, required by
225 // CreateMessage<T>(Arena* arena, Args&&... args), is as follows:
226 //
227 // - The type T must have (at least) two constructors: a constructor callable
228 // with `args` (without `arena`), called when a T is allocated on the heap;
229 // and a constructor callable with `Arena* arena, Args&&... args`, called when
230 // a T is allocated on an arena. If the second constructor is called with a
231 // NULL arena pointer, it must be equivalent to invoking the first
232 // (`args`-only) constructor.
233 //
234 // - The type T must have a particular type trait: a nested type
235 // |InternalArenaConstructable_|. This is usually a typedef to |void|. If no
236 // such type trait exists, then the instantiation CreateMessage<T> will fail
237 // to compile.
238 //
239 // - The type T *may* have the type trait |DestructorSkippable_|. If this type
240 // trait is present in the type, then its destructor will not be called if and
241 // only if it was passed a non-NULL arena pointer. If this type trait is not
242 // present on the type, then its destructor is always called when the
243 // containing arena is destroyed.
244 //
245 // This protocol is implemented by all arena-enabled proto2 message classes as
246 // well as protobuf container types like RepeatedPtrField and Map. The protocol
247 // is internal to protobuf and is not guaranteed to be stable. Non-proto types
248 // should not rely on this protocol.
249 class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
250 public:
251 // Default constructor with sensible default options, tuned for average
252 // use-cases.
253 inline Arena() : impl_() {}
254
255 // Construct an arena with default options, except for the supplied
256 // initial block. It is more efficient to use this constructor
257 // instead of passing ArenaOptions if the only configuration needed
258 // by the caller is supplying an initial block.
259 inline Arena(char* initial_block, size_t initial_block_size)
260 : impl_(initial_block, initial_block_size) {}
261
262 // Arena constructor taking custom options. See ArenaOptions above for
263 // descriptions of the options available.
264 explicit Arena(const ArenaOptions& options)
265 : impl_(options.initial_block, options.initial_block_size,
266 options.AllocationPolicy()) {}
267
268 // Block overhead. Use this as a guide for how much to over-allocate the
269 // initial block if you want an allocation of size N to fit inside it.
270 //
271 // WARNING: if you allocate multiple objects, it is difficult to guarantee
272 // that a series of allocations will fit in the initial block, especially if
273 // Arena changes its alignment guarantees in the future!
274 static const size_t kBlockOverhead =
275 internal::ThreadSafeArena::kBlockHeaderSize +
276 internal::ThreadSafeArena::kSerialArenaSize;
277
278 inline ~Arena() {}
279
280 // TODO(protobuf-team): Fix callers to use constructor and delete this method.
281 void Init(const ArenaOptions&) {}
282
283 // API to create proto2 message objects on the arena. If the arena passed in
284 // is NULL, then a heap allocated object is returned. Type T must be a message
285 // defined in a .proto file with cc_enable_arenas set to true, otherwise a
286 // compilation error will occur.
287 //
288 // RepeatedField and RepeatedPtrField may also be instantiated directly on an
289 // arena with this method.
290 //
291 // This function also accepts any type T that satisfies the arena message
292 // allocation protocol, documented above.
293 template <typename T, typename... Args>
294 PROTOBUF_ALWAYS_INLINE static T* CreateMessage(Arena* arena, Args&&... args) {
295 static_assert(
296 InternalHelper<T>::is_arena_constructable::value,
297 "CreateMessage can only construct types that are ArenaConstructable");
298 // We must delegate to CreateMaybeMessage() and NOT CreateMessageInternal()
299 // because protobuf generated classes specialize CreateMaybeMessage() and we
300 // need to use that specialization for code size reasons.
301 return Arena::CreateMaybeMessage<T>(arena, std::forward<Args>(args)...);
302 }
303
304 // API to create any objects on the arena. Note that only the object will
305 // be created on the arena; the underlying ptrs (in case of a proto2 message)
306 // will be still heap allocated. Proto messages should usually be allocated
307 // with CreateMessage<T>() instead.
308 //
309 // Note that even if T satisfies the arena message construction protocol
310 // (InternalArenaConstructable_ trait and optional DestructorSkippable_
311 // trait), as described above, this function does not follow the protocol;
312 // instead, it treats T as a black-box type, just as if it did not have these
313 // traits. Specifically, T's constructor arguments will always be only those
314 // passed to Create<T>() -- no additional arena pointer is implicitly added.
315 // Furthermore, the destructor will always be called at arena destruction time
316 // (unless the destructor is trivial). Hence, from T's point of view, it is as
317 // if the object were allocated on the heap (except that the underlying memory
318 // is obtained from the arena).
319 template <typename T, typename... Args>
320 PROTOBUF_NDEBUG_INLINE static T* Create(Arena* arena, Args&&... args) {
321 return CreateInternal<T>(arena, std::is_convertible<T*, MessageLite*>(),
322 std::forward<Args>(args)...);
323 }
324
325 // Create an array of object type T on the arena *without* invoking the
326 // constructor of T. If `arena` is null, then the return value should be freed
327 // with `delete[] x;` (or `::operator delete[](x);`).
328 // To ensure safe uses, this function checks at compile time
329 // (when compiled as C++11) that T is trivially default-constructible and
330 // trivially destructible.
331 template <typename T>
332 PROTOBUF_NDEBUG_INLINE static T* CreateArray(Arena* arena,
333 size_t num_elements) {
334 static_assert(std::is_trivial<T>::value,
335 "CreateArray requires a trivially constructible type");
336 static_assert(std::is_trivially_destructible<T>::value,
337 "CreateArray requires a trivially destructible type");
338 GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
339 << "Requested size is too large to fit into size_t.";
340 if (arena == NULL) {
341 return static_cast<T*>(::operator new[](num_elements * sizeof(T)));
342 } else {
343 return arena->CreateInternalRawArray<T>(num_elements);
344 }
345 }
346
347 // The following are routines are for monitoring. They will approximate the
348 // total sum allocated and used memory, but the exact value is an
349 // implementation deal. For instance allocated space depends on growth
350 // policies. Do not use these in unit tests.
351 // Returns the total space allocated by the arena, which is the sum of the
352 // sizes of the underlying blocks.
353 uint64 SpaceAllocated() const { return impl_.SpaceAllocated(); }
354 // Returns the total space used by the arena. Similar to SpaceAllocated but
355 // does not include free space and block overhead. The total space returned
356 // may not include space used by other threads executing concurrently with
357 // the call to this method.
358 uint64 SpaceUsed() const { return impl_.SpaceUsed(); }
359
360 // Frees all storage allocated by this arena after calling destructors
361 // registered with OwnDestructor() and freeing objects registered with Own().
362 // Any objects allocated on this arena are unusable after this call. It also
363 // returns the total space used by the arena which is the sums of the sizes
364 // of the allocated blocks. This method is not thread-safe.
365 uint64 Reset() { return impl_.Reset(); }
366
367 // Adds |object| to a list of heap-allocated objects to be freed with |delete|
368 // when the arena is destroyed or reset.
369 template <typename T>
370 PROTOBUF_ALWAYS_INLINE void Own(T* object) {
371 OwnInternal(object, std::is_convertible<T*, MessageLite*>());
372 }
373
374 // Adds |object| to a list of objects whose destructors will be manually
375 // called when the arena is destroyed or reset. This differs from Own() in
376 // that it does not free the underlying memory with |delete|; hence, it is
377 // normally only used for objects that are placement-newed into
378 // arena-allocated memory.
379 template <typename T>
380 PROTOBUF_ALWAYS_INLINE void OwnDestructor(T* object) {
381 if (object != NULL) {
382 impl_.AddCleanup(object, &internal::arena_destruct_object<T>);
383 }
384 }
385
386 // Adds a custom member function on an object to the list of destructors that
387 // will be manually called when the arena is destroyed or reset. This differs
388 // from OwnDestructor() in that any member function may be specified, not only
389 // the class destructor.
390 PROTOBUF_ALWAYS_INLINE void OwnCustomDestructor(void* object,
391 void (*destruct)(void*)) {
392 impl_.AddCleanup(object, destruct);
393 }
394
395 // Retrieves the arena associated with |value| if |value| is an arena-capable
396 // message, or NULL otherwise. If possible, the call resolves at compile time.
397 // Note that we can often devirtualize calls to `value->GetArena()` so usually
398 // calling this method is unnecessary.
399 template <typename T>
400 PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) {
401 return GetArenaInternal(value);
402 }
403
404 template <typename T>
405 class InternalHelper {
406 public:
407 // Provides access to protected GetOwningArena to generated messages.
408 static Arena* GetOwningArena(const T* p) { return p->GetOwningArena(); }
409
410 // Provides access to protected GetArenaForAllocation to generated messages.
411 static Arena* GetArenaForAllocation(const T* p) {
412 return GetArenaForAllocationInternal(
413 p, std::is_convertible<T*, MessageLite*>());
414 }
415
416 private:
417 static Arena* GetArenaForAllocationInternal(
418 const T* p, std::true_type /*is_derived_from<MessageLite>*/) {
419 return p->GetArenaForAllocation();
420 }
421
422 static Arena* GetArenaForAllocationInternal(
423 const T* p, std::false_type /*is_derived_from<MessageLite>*/) {
424 return GetArenaForAllocationForNonMessage(
425 p, typename is_arena_constructable::type());
426 }
427
428 static Arena* GetArenaForAllocationForNonMessage(
429 const T* p, std::true_type /*is_arena_constructible*/) {
430 return p->GetArena();
431 }
432
433 static Arena* GetArenaForAllocationForNonMessage(
434 const T* p, std::false_type /*is_arena_constructible*/) {
435 return GetArenaForAllocationForNonMessageNonArenaConstructible(
436 p, typename has_get_arena::type());
437 }
438
439 static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible(
440 const T* p, std::true_type /*has_get_arena*/) {
441 return p->GetArena();
442 }
443
444 static Arena* GetArenaForAllocationForNonMessageNonArenaConstructible(
445 const T* p, std::false_type /*has_get_arena*/) {
446 return nullptr;
447 }
448
449 template <typename U>
450 static char DestructorSkippable(const typename U::DestructorSkippable_*);
451 template <typename U>
452 static double DestructorSkippable(...);
453
454 typedef std::integral_constant<
455 bool, sizeof(DestructorSkippable<T>(static_cast<const T*>(0))) ==
456 sizeof(char) ||
457 std::is_trivially_destructible<T>::value>
458 is_destructor_skippable;
459
460 template <typename U>
461 static char ArenaConstructable(
462 const typename U::InternalArenaConstructable_*);
463 template <typename U>
464 static double ArenaConstructable(...);
465
466 typedef std::integral_constant<bool, sizeof(ArenaConstructable<T>(
467 static_cast<const T*>(0))) ==
468 sizeof(char)>
469 is_arena_constructable;
470
471 template <typename U,
472 typename std::enable_if<
473 std::is_same<Arena*, decltype(std::declval<const U>()
474 .GetArena())>::value,
475 int>::type = 0>
476 static char HasGetArena(decltype(&U::GetArena));
477 template <typename U>
478 static double HasGetArena(...);
479
480 typedef std::integral_constant<bool, sizeof(HasGetArena<T>(nullptr)) ==
481 sizeof(char)>
482 has_get_arena;
483
484 template <typename... Args>
485 static T* Construct(void* ptr, Args&&... args) {
486 return new (ptr) T(std::forward<Args>(args)...);
487 }
488
489 static T* New() {
490 return new T(nullptr);
491 }
492
493 static Arena* GetArena(const T* p) { return p->GetArena(); }
494
495 friend class Arena;
496 friend class TestUtil::ReflectionTester;
497 };
498
499 // Helper typetraits that indicates support for arenas in a type T at compile
500 // time. This is public only to allow construction of higher-level templated
501 // utilities.
502 //
503 // is_arena_constructable<T>::value is true if the message type T has arena
504 // support enabled, and false otherwise.
505 //
506 // is_destructor_skippable<T>::value is true if the message type T has told
507 // the arena that it is safe to skip the destructor, and false otherwise.
508 //
509 // This is inside Arena because only Arena has the friend relationships
510 // necessary to see the underlying generated code traits.
511 template <typename T>
512 struct is_arena_constructable : InternalHelper<T>::is_arena_constructable {};
513 template <typename T>
514 struct is_destructor_skippable : InternalHelper<T>::is_destructor_skippable {
515 };
516
517 private:
518 internal::ThreadSafeArena impl_;
519
520 template <typename T>
521 struct has_get_arena : InternalHelper<T>::has_get_arena {};
522
523 template <typename T, typename... Args>
524 PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena,
525 Args&&... args) {
526 static_assert(
527 InternalHelper<T>::is_arena_constructable::value,
528 "CreateMessage can only construct types that are ArenaConstructable");
529 if (arena == NULL) {
530 return new T(nullptr, std::forward<Args>(args)...);
531 } else {
532 return arena->DoCreateMessage<T>(std::forward<Args>(args)...);
533 }
534 }
535
536 // This specialization for no arguments is necessary, because its behavior is
537 // slightly different. When the arena pointer is nullptr, it calls T()
538 // instead of T(nullptr).
539 template <typename T>
540 PROTOBUF_NDEBUG_INLINE static T* CreateMessageInternal(Arena* arena) {
541 static_assert(
542 InternalHelper<T>::is_arena_constructable::value,
543 "CreateMessage can only construct types that are ArenaConstructable");
544 if (arena == NULL) {
545 // Generated arena constructor T(Arena*) is protected. Call via
546 // InternalHelper.
547 return InternalHelper<T>::New();
548 } else {
549 return arena->DoCreateMessage<T>();
550 }
551 }
552
553 // Allocate and also optionally call collector with the allocated type info
554 // when allocation recording is enabled.
555 PROTOBUF_NDEBUG_INLINE void* AllocateInternal(size_t size, size_t align,
556 void (*destructor)(void*),
557 const std::type_info* type) {
558 // Monitor allocation if needed.
559 if (destructor == nullptr) {
560 return AllocateAlignedWithHook(size, align, type);
561 } else {
562 if (align <= 8) {
563 auto res = AllocateAlignedWithCleanup(internal::AlignUpTo8(size), type);
564 res.second->elem = res.first;
565 res.second->cleanup = destructor;
566 return res.first;
567 } else {
568 auto res = AllocateAlignedWithCleanup(size + align - 8, type);
569 auto ptr = internal::AlignTo(res.first, align);
570 res.second->elem = ptr;
571 res.second->cleanup = destructor;
572 return ptr;
573 }
574 }
575 }
576
577 // CreateMessage<T> requires that T supports arenas, but this private method
578 // works whether or not T supports arenas. These are not exposed to user code
579 // as it can cause confusing API usages, and end up having double free in
580 // user code. These are used only internally from LazyField and Repeated
581 // fields, since they are designed to work in all mode combinations.
582 template <typename Msg, typename... Args>
583 PROTOBUF_ALWAYS_INLINE static Msg* DoCreateMaybeMessage(Arena* arena,
584 std::true_type,
585 Args&&... args) {
586 return CreateMessageInternal<Msg>(arena, std::forward<Args>(args)...);
587 }
588
589 template <typename T, typename... Args>
590 PROTOBUF_ALWAYS_INLINE static T* DoCreateMaybeMessage(Arena* arena,
591 std::false_type,
592 Args&&... args) {
593 return Create<T>(arena, std::forward<Args>(args)...);
594 }
595
596 template <typename T, typename... Args>
597 PROTOBUF_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena,
598 Args&&... args) {
599 return DoCreateMaybeMessage<T>(arena, is_arena_constructable<T>(),
600 std::forward<Args>(args)...);
601 }
602
603 // Just allocate the required size for the given type assuming the
604 // type has a trivial constructor.
605 template <typename T>
606 PROTOBUF_NDEBUG_INLINE T* CreateInternalRawArray(size_t num_elements) {
607 GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
608 << "Requested size is too large to fit into size_t.";
609 // We count on compiler to realize that if sizeof(T) is a multiple of
610 // 8 AlignUpTo can be elided.
611 const size_t n = sizeof(T) * num_elements;
612 return static_cast<T*>(
613 AllocateAlignedWithHook(n, alignof(T), RTTI_TYPE_ID(T)));
614 }
615
616 template <typename T, typename... Args>
617 PROTOBUF_NDEBUG_INLINE T* DoCreateMessage(Args&&... args) {
618 return InternalHelper<T>::Construct(
619 AllocateInternal(sizeof(T), alignof(T),
620 internal::ObjectDestructor<
621 InternalHelper<T>::is_destructor_skippable::value,
622 T>::destructor,
623 RTTI_TYPE_ID(T)),
624 this, std::forward<Args>(args)...);
625 }
626
627 // CreateInArenaStorage is used to implement map field. Without it,
628 // Map need to call generated message's protected arena constructor,
629 // which needs to declare Map as friend of generated message.
630 template <typename T, typename... Args>
631 static void CreateInArenaStorage(T* ptr, Arena* arena, Args&&... args) {
632 CreateInArenaStorageInternal(ptr, arena,
633 typename is_arena_constructable<T>::type(),
634 std::forward<Args>(args)...);
635 RegisterDestructorInternal(
636 ptr, arena,
637 typename InternalHelper<T>::is_destructor_skippable::type());
638 }
639
640 template <typename T, typename... Args>
641 static void CreateInArenaStorageInternal(T* ptr, Arena* arena,
642 std::true_type, Args&&... args) {
643 InternalHelper<T>::Construct(ptr, arena, std::forward<Args>(args)...);
644 }
645 template <typename T, typename... Args>
646 static void CreateInArenaStorageInternal(T* ptr, Arena* /* arena */,
647 std::false_type, Args&&... args) {
648 new (ptr) T(std::forward<Args>(args)...);
649 }
650
651 template <typename T>
652 static void RegisterDestructorInternal(T* /* ptr */, Arena* /* arena */,
653 std::true_type) {}
654 template <typename T>
655 static void RegisterDestructorInternal(T* ptr, Arena* arena,
656 std::false_type) {
657 arena->OwnDestructor(ptr);
658 }
659
660 // These implement Create(). The second parameter has type 'true_type' if T is
661 // a subtype of Message and 'false_type' otherwise.
662 template <typename T, typename... Args>
663 PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::true_type,
664 Args&&... args) {
665 if (arena == nullptr) {
666 return new T(std::forward<Args>(args)...);
667 } else {
668 auto destructor =
669 internal::ObjectDestructor<std::is_trivially_destructible<T>::value,
670 T>::destructor;
671 T* result =
672 new (arena->AllocateInternal(sizeof(T), alignof(T), destructor,
673 RTTI_TYPE_ID(T)))
674 T(std::forward<Args>(args)...);
675 return result;
676 }
677 }
678 template <typename T, typename... Args>
679 PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena, std::false_type,
680 Args&&... args) {
681 if (arena == nullptr) {
682 return new T(std::forward<Args>(args)...);
683 } else {
684 auto destructor =
685 internal::ObjectDestructor<std::is_trivially_destructible<T>::value,
686 T>::destructor;
687 return new (arena->AllocateInternal(sizeof(T), alignof(T), destructor,
688 RTTI_TYPE_ID(T)))
689 T(std::forward<Args>(args)...);
690 }
691 }
692
693 // These implement Own(), which registers an object for deletion (destructor
694 // call and operator delete()). The second parameter has type 'true_type' if T
695 // is a subtype of Message and 'false_type' otherwise. Collapsing
696 // all template instantiations to one for generic Message reduces code size,
697 // using the virtual destructor instead.
698 template <typename T>
699 PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) {
700 if (object != NULL) {
701 impl_.AddCleanup(object, &internal::arena_delete_object<MessageLite>);
702 }
703 }
704 template <typename T>
705 PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) {
706 if (object != NULL) {
707 impl_.AddCleanup(object, &internal::arena_delete_object<T>);
708 }
709 }
710
711 // Implementation for GetArena(). Only message objects with
712 // InternalArenaConstructable_ tags can be associated with an arena, and such
713 // objects must implement a GetArena() method.
714 template <typename T, typename std::enable_if<
715 is_arena_constructable<T>::value, int>::type = 0>
716 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
717 return InternalHelper<T>::GetArena(value);
718 }
719 template <typename T,
720 typename std::enable_if<!is_arena_constructable<T>::value &&
721 has_get_arena<T>::value,
722 int>::type = 0>
723 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
724 return value->GetArena();
725 }
726 template <typename T,
727 typename std::enable_if<!is_arena_constructable<T>::value &&
728 !has_get_arena<T>::value,
729 int>::type = 0>
730 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
731 (void)value;
732 return nullptr;
733 }
734
735 template <typename T>
736 PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArena(const T* value) {
737 return GetOwningArenaInternal(
738 value, std::is_convertible<T*, MessageLite*>());
739 }
740
741 // Implementation for GetOwningArena(). All and only message objects have
742 // GetOwningArena() method.
743 template <typename T>
744 PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal(
745 const T* value, std::true_type) {
746 return InternalHelper<T>::GetOwningArena(value);
747 }
748 template <typename T>
749 PROTOBUF_ALWAYS_INLINE static Arena* GetOwningArenaInternal(
750 const T* /* value */, std::false_type) {
751 return nullptr;
752 }
753
754 // For friends of arena.
755 void* AllocateAligned(size_t n, size_t align = 8) {
756 if (align <= 8) {
757 return AllocateAlignedNoHook(internal::AlignUpTo8(n));
758 } else {
759 // We are wasting space by over allocating align - 8 bytes. Compared
760 // to a dedicated function that takes current alignment in consideration.
761 // Such a scheme would only waste (align - 8)/2 bytes on average, but
762 // requires a dedicated function in the outline arena allocation
763 // functions. Possibly re-evaluate tradeoffs later.
764 return internal::AlignTo(AllocateAlignedNoHook(n + align - 8), align);
765 }
766 }
767
768 void* AllocateAlignedWithHook(size_t n, size_t align,
769 const std::type_info* type) {
770 if (align <= 8) {
771 return AllocateAlignedWithHook(internal::AlignUpTo8(n), type);
772 } else {
773 // We are wasting space by over allocating align - 8 bytes. Compared
774 // to a dedicated function that takes current alignment in consideration.
775 // Such a schemee would only waste (align - 8)/2 bytes on average, but
776 // requires a dedicated function in the outline arena allocation
777 // functions. Possibly re-evaluate tradeoffs later.
778 return internal::AlignTo(AllocateAlignedWithHook(n + align - 8, type),
779 align);
780 }
781 }
782
783 void* AllocateAlignedNoHook(size_t n);
784 void* AllocateAlignedWithHook(size_t n, const std::type_info* type);
785 std::pair<void*, internal::SerialArena::CleanupNode*>
786 AllocateAlignedWithCleanup(size_t n, const std::type_info* type);
787
788 template <typename Type>
789 friend class internal::GenericTypeHandler;
790 friend struct internal::ArenaStringPtr; // For AllocateAligned.
791 friend class internal::LazyField; // For CreateMaybeMessage.
792 friend class internal::EpsCopyInputStream; // For parser performance
793 friend class MessageLite;
794 template <typename Key, typename T>
795 friend class Map;
796 };
797
798 // Defined above for supporting environments without RTTI.
799 #undef RTTI_TYPE_ID
800
801 } // namespace protobuf
802 } // namespace google
803
804 #include <google/protobuf/port_undef.inc>
805
806 #endif // GOOGLE_PROTOBUF_ARENA_H__
807