1// Protocol Buffers - Google's data interchange format
2// Copyright 2008 Google Inc. All rights reserved.
3// https://developers.google.com/protocol-buffers/
4//
5// Redistribution and use in source and binary forms, with or without
6// modification, are permitted provided that the following conditions are
7// met:
8//
9// * Redistributions of source code must retain the above copyright
10// notice, this list of conditions and the following disclaimer.
11// * Redistributions in binary form must reproduce the above
12// copyright notice, this list of conditions and the following disclaimer
13// in the documentation and/or other materials provided with the
14// distribution.
15// * Neither the name of Google Inc. nor the names of its
16// contributors may be used to endorse or promote products derived from
17// this software without specific prior written permission.
18//
19// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31// This file defines an Arena allocator for better allocation performance.
32
33#ifndef GOOGLE_PROTOBUF_ARENA_H__
34#define GOOGLE_PROTOBUF_ARENA_H__
35
36
37#include <limits>
38#include <type_traits>
39#include <utility>
40#ifdef max
41#undef max // Visual Studio defines this macro
42#endif
43#if defined(_MSC_VER) && !defined(_LIBCPP_STD_VER) && !_HAS_EXCEPTIONS
44// Work around bugs in MSVC <typeinfo> header when _HAS_EXCEPTIONS=0.
45#include <exception>
46#include <typeinfo>
47namespace std {
48using type_info = ::type_info;
49}
50#else
51#include <typeinfo>
52#endif
53
54#include <type_traits>
55#include <google/protobuf/arena_impl.h>
56#include <google/protobuf/port.h>
57
58#include <google/protobuf/port_def.inc>
59
60#ifdef SWIG
61#error "You cannot SWIG proto headers"
62#endif
63
64namespace google {
65namespace protobuf {
66
67struct ArenaOptions; // defined below
68
69} // namespace protobuf
70} // namespace google
71
72namespace google {
73namespace protobuf {
74
75class Arena; // defined below
76class Message; // defined in message.h
77class MessageLite;
78template <typename Key, typename T>
79class Map;
80
81namespace arena_metrics {
82
83void EnableArenaMetrics(ArenaOptions* options);
84
85} // namespace arena_metrics
86
87namespace internal {
88
89struct ArenaStringPtr; // defined in arenastring.h
90class LazyField; // defined in lazy_field.h
91class EpsCopyInputStream; // defined in parse_context.h
92
93template <typename Type>
94class GenericTypeHandler; // defined in repeated_field.h
95
96// Templated cleanup methods.
97template <typename T>
98void arena_destruct_object(void* object) {
99 reinterpret_cast<T*>(object)->~T();
100}
101template <typename T>
102void arena_delete_object(void* object) {
103 delete reinterpret_cast<T*>(object);
104}
105inline void arena_free(void* object, size_t size) {
106#if defined(__GXX_DELETE_WITH_SIZE__) || defined(__cpp_sized_deallocation)
107 ::operator delete(object, size);
108#else
109 (void)size;
110 ::operator delete(object);
111#endif
112}
113
114} // namespace internal
115
116// ArenaOptions provides optional additional parameters to arena construction
117// that control its block-allocation behavior.
118struct ArenaOptions {
119 // This defines the size of the first block requested from the system malloc.
120 // Subsequent block sizes will increase in a geometric series up to a maximum.
121 size_t start_block_size;
122
123 // This defines the maximum block size requested from system malloc (unless an
124 // individual arena allocation request occurs with a size larger than this
125 // maximum). Requested block sizes increase up to this value, then remain
126 // here.
127 size_t max_block_size;
128
129 // An initial block of memory for the arena to use, or NULL for none. If
130 // provided, the block must live at least as long as the arena itself. The
131 // creator of the Arena retains ownership of the block after the Arena is
132 // destroyed.
133 char* initial_block;
134
135 // The size of the initial block, if provided.
136 size_t initial_block_size;
137
138 // A function pointer to an alloc method that returns memory blocks of size
139 // requested. By default, it contains a ptr to the malloc function.
140 //
141 // NOTE: block_alloc and dealloc functions are expected to behave like
142 // malloc and free, including Asan poisoning.
143 void* (*block_alloc)(size_t);
144 // A function pointer to a dealloc method that takes ownership of the blocks
145 // from the arena. By default, it contains a ptr to a wrapper function that
146 // calls free.
147 void (*block_dealloc)(void*, size_t);
148
149 ArenaOptions()
150 : start_block_size(kDefaultStartBlockSize),
151 max_block_size(kDefaultMaxBlockSize),
152 initial_block(NULL),
153 initial_block_size(0),
154 block_alloc(&::operator new),
155 block_dealloc(&internal::arena_free),
156 on_arena_init(NULL),
157 on_arena_reset(NULL),
158 on_arena_destruction(NULL),
159 on_arena_allocation(NULL) {}
160
161 private:
162 // Hooks for adding external functionality such as user-specific metrics
163 // collection, specific debugging abilities, etc.
164 // Init hook (if set) will always be called at Arena init time. Init hook may
165 // return a pointer to a cookie to be stored in the arena. Reset and
166 // destruction hooks will then be called with the same cookie pointer. This
167 // allows us to save an external object per arena instance and use it on the
168 // other hooks (Note: If init hook returns NULL, the other hooks will NOT be
169 // called on this arena instance).
170 // on_arena_reset and on_arena_destruction also receive the space used in the
171 // arena just before the reset.
172 void* (*on_arena_init)(Arena* arena);
173 void (*on_arena_reset)(Arena* arena, void* cookie, uint64 space_used);
174 void (*on_arena_destruction)(Arena* arena, void* cookie, uint64 space_used);
175
176 // type_info is promised to be static - its lifetime extends to
177 // match program's lifetime (It is given by typeid operator).
178 // Note: typeid(void) will be passed as allocated_type every time we
179 // intentionally want to avoid monitoring an allocation. (i.e. internal
180 // allocations for managing the arena)
181 void (*on_arena_allocation)(const std::type_info* allocated_type,
182 uint64 alloc_size, void* cookie);
183
184 // Constants define default starting block size and max block size for
185 // arena allocator behavior -- see descriptions above.
186 static const size_t kDefaultStartBlockSize = 256;
187 static const size_t kDefaultMaxBlockSize = 8192;
188
189 friend void arena_metrics::EnableArenaMetrics(ArenaOptions*);
190 friend class Arena;
191 friend class ArenaOptionsTestFriend;
192};
193
194// Support for non-RTTI environments. (The metrics hooks API uses type
195// information.)
196#if PROTOBUF_RTTI
197#define RTTI_TYPE_ID(type) (&typeid(type))
198#else
199#define RTTI_TYPE_ID(type) (NULL)
200#endif
201
202// Arena allocator. Arena allocation replaces ordinary (heap-based) allocation
203// with new/delete, and improves performance by aggregating allocations into
204// larger blocks and freeing allocations all at once. Protocol messages are
205// allocated on an arena by using Arena::CreateMessage<T>(Arena*), below, and
206// are automatically freed when the arena is destroyed.
207//
208// This is a thread-safe implementation: multiple threads may allocate from the
209// arena concurrently. Destruction is not thread-safe and the destructing
210// thread must synchronize with users of the arena first.
211//
212// An arena provides two allocation interfaces: CreateMessage<T>, which works
213// for arena-enabled proto2 message types as well as other types that satisfy
214// the appropriate protocol (described below), and Create<T>, which works for
215// any arbitrary type T. CreateMessage<T> is better when the type T supports it,
216// because this interface (i) passes the arena pointer to the created object so
217// that its sub-objects and internal allocations can use the arena too, and (ii)
218// elides the object's destructor call when possible. Create<T> does not place
219// any special requirements on the type T, and will invoke the object's
220// destructor when the arena is destroyed.
221//
222// The arena message allocation protocol, required by
223// CreateMessage<T>(Arena* arena, Args&&... args), is as follows:
224//
225// - The type T must have (at least) two constructors: a constructor callable
226// with `args` (without `arena`), called when a T is allocated on the heap;
227// and a constructor callable with `Arena* arena, Args&&... args`, called when
228// a T is allocated on an arena. If the second constructor is called with a
229// NULL arena pointer, it must be equivalent to invoking the first
230// (`args`-only) constructor.
231//
232// - The type T must have a particular type trait: a nested type
233// |InternalArenaConstructable_|. This is usually a typedef to |void|. If no
234// such type trait exists, then the instantiation CreateMessage<T> will fail
235// to compile.
236//
237// - The type T *may* have the type trait |DestructorSkippable_|. If this type
238// trait is present in the type, then its destructor will not be called if and
239// only if it was passed a non-NULL arena pointer. If this type trait is not
240// present on the type, then its destructor is always called when the
241// containing arena is destroyed.
242//
243// This protocol is implemented by all arena-enabled proto2 message classes as
244// well as protobuf container types like RepeatedPtrField and Map. The protocol
245// is internal to protobuf and is not guaranteed to be stable. Non-proto types
246// should not rely on this protocol.
247class PROTOBUF_EXPORT PROTOBUF_ALIGNAS(8) Arena final {
248 public:
249 // Arena constructor taking custom options. See ArenaOptions below for
250 // descriptions of the options available.
251 explicit Arena(const ArenaOptions& options) : impl_(options) {
252 Init(options);
253 }
254
255 // Block overhead. Use this as a guide for how much to over-allocate the
256 // initial block if you want an allocation of size N to fit inside it.
257 //
258 // WARNING: if you allocate multiple objects, it is difficult to guarantee
259 // that a series of allocations will fit in the initial block, especially if
260 // Arena changes its alignment guarantees in the future!
261 static const size_t kBlockOverhead = internal::ArenaImpl::kBlockHeaderSize +
262 internal::ArenaImpl::kSerialArenaSize;
263
264 // Default constructor with sensible default options, tuned for average
265 // use-cases.
266 Arena() : impl_(ArenaOptions()) { Init(options: ArenaOptions()); }
267
268 ~Arena() {
269 if (hooks_cookie_) {
270 CallDestructorHooks();
271 }
272 }
273
274 void Init(const ArenaOptions& options) {
275 on_arena_allocation_ = options.on_arena_allocation;
276 on_arena_reset_ = options.on_arena_reset;
277 on_arena_destruction_ = options.on_arena_destruction;
278 // Call the initialization hook
279 if (options.on_arena_init != NULL) {
280 hooks_cookie_ = options.on_arena_init(this);
281 } else {
282 hooks_cookie_ = NULL;
283 }
284 }
285
286 // API to create proto2 message objects on the arena. If the arena passed in
287 // is NULL, then a heap allocated object is returned. Type T must be a message
288 // defined in a .proto file with cc_enable_arenas set to true, otherwise a
289 // compilation error will occur.
290 //
291 // RepeatedField and RepeatedPtrField may also be instantiated directly on an
292 // arena with this method.
293 //
294 // This function also accepts any type T that satisfies the arena message
295 // allocation protocol, documented above.
296 template <typename T, typename... Args>
297 PROTOBUF_ALWAYS_INLINE static T* CreateMessage(Arena* arena, Args&&... args) {
298 static_assert(
299 InternalHelper<T>::is_arena_constructable::value,
300 "CreateMessage can only construct types that are ArenaConstructable");
301 // We must delegate to CreateMaybeMessage() and NOT CreateMessageInternal()
302 // because protobuf generated classes specialize CreateMaybeMessage() and we
303 // need to use that specialization for code size reasons.
304 return Arena::CreateMaybeMessage<T>(arena, std::forward<Args>(args)...);
305 }
306
307 // API to create any objects on the arena. Note that only the object will
308 // be created on the arena; the underlying ptrs (in case of a proto2 message)
309 // will be still heap allocated. Proto messages should usually be allocated
310 // with CreateMessage<T>() instead.
311 //
312 // Note that even if T satisfies the arena message construction protocol
313 // (InternalArenaConstructable_ trait and optional DestructorSkippable_
314 // trait), as described above, this function does not follow the protocol;
315 // instead, it treats T as a black-box type, just as if it did not have these
316 // traits. Specifically, T's constructor arguments will always be only those
317 // passed to Create<T>() -- no additional arena pointer is implicitly added.
318 // Furthermore, the destructor will always be called at arena destruction time
319 // (unless the destructor is trivial). Hence, from T's point of view, it is as
320 // if the object were allocated on the heap (except that the underlying memory
321 // is obtained from the arena).
322 template <typename T, typename... Args>
323 PROTOBUF_ALWAYS_INLINE static T* Create(Arena* arena, Args&&... args) {
324 return CreateNoMessage<T>(arena, is_arena_constructable<T>(),
325 std::forward<Args>(args)...);
326 }
327
328 // Create an array of object type T on the arena *without* invoking the
329 // constructor of T. If `arena` is null, then the return value should be freed
330 // with `delete[] x;` (or `::operator delete[](x);`).
331 // To ensure safe uses, this function checks at compile time
332 // (when compiled as C++11) that T is trivially default-constructible and
333 // trivially destructible.
334 template <typename T>
335 PROTOBUF_ALWAYS_INLINE static T* CreateArray(Arena* arena,
336 size_t num_elements) {
337 static_assert(std::is_pod<T>::value,
338 "CreateArray requires a trivially constructible type");
339 static_assert(std::is_trivially_destructible<T>::value,
340 "CreateArray requires a trivially destructible type");
341 GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
342 << "Requested size is too large to fit into size_t.";
343 if (arena == NULL) {
344 return static_cast<T*>(::operator new[](num_elements * sizeof(T)));
345 } else {
346 return arena->CreateInternalRawArray<T>(num_elements);
347 }
348 }
349
350 // Returns the total space allocated by the arena, which is the sum of the
351 // sizes of the underlying blocks. This method is relatively fast; a counter
352 // is kept as blocks are allocated.
353 uint64 SpaceAllocated() const { return impl_.SpaceAllocated(); }
354 // Returns the total space used by the arena. Similar to SpaceAllocated but
355 // does not include free space and block overhead. The total space returned
356 // may not include space used by other threads executing concurrently with
357 // the call to this method.
358 uint64 SpaceUsed() const { return impl_.SpaceUsed(); }
359
360 // Frees all storage allocated by this arena after calling destructors
361 // registered with OwnDestructor() and freeing objects registered with Own().
362 // Any objects allocated on this arena are unusable after this call. It also
363 // returns the total space used by the arena which is the sums of the sizes
364 // of the allocated blocks. This method is not thread-safe.
365 PROTOBUF_NOINLINE uint64 Reset() {
366 // Call the reset hook
367 if (on_arena_reset_ != NULL) {
368 on_arena_reset_(this, hooks_cookie_, impl_.SpaceAllocated());
369 }
370 return impl_.Reset();
371 }
372
373 // Adds |object| to a list of heap-allocated objects to be freed with |delete|
374 // when the arena is destroyed or reset.
375 template <typename T>
376 PROTOBUF_NOINLINE void Own(T* object) {
377 OwnInternal(object, std::is_convertible<T*, Message*>());
378 }
379
380 // Adds |object| to a list of objects whose destructors will be manually
381 // called when the arena is destroyed or reset. This differs from Own() in
382 // that it does not free the underlying memory with |delete|; hence, it is
383 // normally only used for objects that are placement-newed into
384 // arena-allocated memory.
385 template <typename T>
386 PROTOBUF_NOINLINE void OwnDestructor(T* object) {
387 if (object != NULL) {
388 impl_.AddCleanup(elem: object, cleanup: &internal::arena_destruct_object<T>);
389 }
390 }
391
392 // Adds a custom member function on an object to the list of destructors that
393 // will be manually called when the arena is destroyed or reset. This differs
394 // from OwnDestructor() in that any member function may be specified, not only
395 // the class destructor.
396 PROTOBUF_NOINLINE void OwnCustomDestructor(void* object,
397 void (*destruct)(void*)) {
398 impl_.AddCleanup(elem: object, cleanup: destruct);
399 }
400
401 // Retrieves the arena associated with |value| if |value| is an arena-capable
402 // message, or NULL otherwise. If possible, the call resolves at compile time.
403 // Note that we can often devirtualize calls to `value->GetArena()` so usually
404 // calling this method is unnecessary.
405 template <typename T>
406 PROTOBUF_ALWAYS_INLINE static Arena* GetArena(const T* value) {
407 return GetArenaInternal(value);
408 }
409
410 template <typename T>
411 class InternalHelper {
412 template <typename U>
413 static char DestructorSkippable(const typename U::DestructorSkippable_*);
414 template <typename U>
415 static double DestructorSkippable(...);
416
417 typedef std::integral_constant<
418 bool, sizeof(DestructorSkippable<T>(static_cast<const T*>(0))) ==
419 sizeof(char) ||
420 std::is_trivially_destructible<T>::value>
421 is_destructor_skippable;
422
423 template <typename U>
424 static char ArenaConstructable(
425 const typename U::InternalArenaConstructable_*);
426 template <typename U>
427 static double ArenaConstructable(...);
428
429 typedef std::integral_constant<bool, sizeof(ArenaConstructable<T>(
430 static_cast<const T*>(0))) ==
431 sizeof(char)>
432 is_arena_constructable;
433
434 template <typename U,
435 typename std::enable_if<
436 std::is_same<Arena*, decltype(std::declval<const U>()
437 .GetArena())>::value,
438 int>::type = 0>
439 static char HasGetArena(decltype(&U::GetArena));
440 template <typename U>
441 static double HasGetArena(...);
442
443 typedef std::integral_constant<bool, sizeof(HasGetArena<T>(nullptr)) ==
444 sizeof(char)>
445 has_get_arena;
446
447 template <typename... Args>
448 static T* Construct(void* ptr, Args&&... args) {
449 return new (ptr) T(std::forward<Args>(args)...);
450 }
451
452 static Arena* GetArena(const T* p) { return p->GetArena(); }
453
454 friend class Arena;
455 };
456
457 // Helper typetraits that indicates support for arenas in a type T at compile
458 // time. This is public only to allow construction of higher-level templated
459 // utilities.
460 //
461 // is_arena_constructable<T>::value is true if the message type T has arena
462 // support enabled, and false otherwise.
463 //
464 // is_destructor_skippable<T>::value is true if the message type T has told
465 // the arena that it is safe to skip the destructor, and false otherwise.
466 //
467 // This is inside Arena because only Arena has the friend relationships
468 // necessary to see the underlying generated code traits.
469 template <typename T>
470 struct is_arena_constructable : InternalHelper<T>::is_arena_constructable {};
471 template <typename T>
472 struct is_destructor_skippable : InternalHelper<T>::is_destructor_skippable {
473 };
474
475 private:
476 template <typename T>
477 struct has_get_arena : InternalHelper<T>::has_get_arena {};
478
479 template <typename T, typename... Args>
480 PROTOBUF_ALWAYS_INLINE static T* CreateMessageInternal(Arena* arena,
481 Args&&... args) {
482 static_assert(
483 InternalHelper<T>::is_arena_constructable::value,
484 "CreateMessage can only construct types that are ArenaConstructable");
485 if (arena == NULL) {
486 return new T(nullptr, std::forward<Args>(args)...);
487 } else {
488 return arena->DoCreateMessage<T>(std::forward<Args>(args)...);
489 }
490 }
491
492 // This specialization for no arguments is necessary, because its behavior is
493 // slightly different. When the arena pointer is nullptr, it calls T()
494 // instead of T(nullptr).
495 template <typename T>
496 PROTOBUF_ALWAYS_INLINE static T* CreateMessageInternal(Arena* arena) {
497 static_assert(
498 InternalHelper<T>::is_arena_constructable::value,
499 "CreateMessage can only construct types that are ArenaConstructable");
500 if (arena == NULL) {
501 return new T();
502 } else {
503 return arena->DoCreateMessage<T>();
504 }
505 }
506
507 template <typename T, typename... Args>
508 PROTOBUF_ALWAYS_INLINE static T* CreateInternal(Arena* arena,
509 Args&&... args) {
510 if (arena == NULL) {
511 return new T(std::forward<Args>(args)...);
512 } else {
513 return arena->DoCreate<T>(std::is_trivially_destructible<T>::value,
514 std::forward<Args>(args)...);
515 }
516 }
517
518 void CallDestructorHooks();
519 void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const;
520 inline void AllocHook(const std::type_info* allocated_type, size_t n) const {
521 if (PROTOBUF_PREDICT_FALSE(hooks_cookie_ != NULL)) {
522 OnArenaAllocation(allocated_type, n);
523 }
524 }
525
526 // Allocate and also optionally call on_arena_allocation callback with the
527 // allocated type info when the hooks are in place in ArenaOptions and
528 // the cookie is not null.
529 template <typename T>
530 PROTOBUF_ALWAYS_INLINE void* AllocateInternal(bool skip_explicit_ownership) {
531 static_assert(alignof(T) <= 8, "T is overaligned, see b/151247138");
532 const size_t n = internal::AlignUpTo8(n: sizeof(T));
533 AllocHook(RTTI_TYPE_ID(T), n);
534 // Monitor allocation if needed.
535 if (skip_explicit_ownership) {
536 return AllocateAlignedNoHook(n);
537 } else {
538 return impl_.AllocateAlignedAndAddCleanup(
539 n, cleanup: &internal::arena_destruct_object<T>);
540 }
541 }
542
543 // CreateMessage<T> requires that T supports arenas, but this private method
544 // works whether or not T supports arenas. These are not exposed to user code
545 // as it can cause confusing API usages, and end up having double free in
546 // user code. These are used only internally from LazyField and Repeated
547 // fields, since they are designed to work in all mode combinations.
548 template <typename Msg, typename... Args>
549 PROTOBUF_ALWAYS_INLINE static Msg* DoCreateMaybeMessage(Arena* arena,
550 std::true_type,
551 Args&&... args) {
552 return CreateMessageInternal<Msg>(arena, std::forward<Args>(args)...);
553 }
554
555 template <typename T, typename... Args>
556 PROTOBUF_ALWAYS_INLINE static T* DoCreateMaybeMessage(Arena* arena,
557 std::false_type,
558 Args&&... args) {
559 return CreateInternal<T>(arena, std::forward<Args>(args)...);
560 }
561
562 template <typename T, typename... Args>
563 PROTOBUF_ALWAYS_INLINE static T* CreateMaybeMessage(Arena* arena,
564 Args&&... args) {
565 return DoCreateMaybeMessage<T>(arena, is_arena_constructable<T>(),
566 std::forward<Args>(args)...);
567 }
568
569 template <typename T, typename... Args>
570 PROTOBUF_ALWAYS_INLINE static T* CreateNoMessage(Arena* arena, std::true_type,
571 Args&&... args) {
572 // User is constructing with Create() despite the fact that T supports arena
573 // construction. In this case we have to delegate to CreateInternal(), and
574 // we can't use any CreateMaybeMessage() specialization that may be defined.
575 return CreateInternal<T>(arena, std::forward<Args>(args)...);
576 }
577
578 template <typename T, typename... Args>
579 PROTOBUF_ALWAYS_INLINE static T* CreateNoMessage(Arena* arena,
580 std::false_type,
581 Args&&... args) {
582 // User is constructing with Create() and the type does not support arena
583 // construction. In this case we can delegate to CreateMaybeMessage() and
584 // use any specialization that may be available for that.
585 return CreateMaybeMessage<T>(arena, std::forward<Args>(args)...);
586 }
587
588 // Just allocate the required size for the given type assuming the
589 // type has a trivial constructor.
590 template <typename T>
591 PROTOBUF_ALWAYS_INLINE T* CreateInternalRawArray(size_t num_elements) {
592 GOOGLE_CHECK_LE(num_elements, std::numeric_limits<size_t>::max() / sizeof(T))
593 << "Requested size is too large to fit into size_t.";
594 const size_t n = internal::AlignUpTo8(n: sizeof(T) * num_elements);
595 // Monitor allocation if needed.
596 AllocHook(RTTI_TYPE_ID(T), n);
597 return static_cast<T*>(AllocateAlignedNoHook(n));
598 }
599
600 template <typename T, typename... Args>
601 PROTOBUF_ALWAYS_INLINE T* DoCreate(bool skip_explicit_ownership,
602 Args&&... args) {
603 return new (AllocateInternal<T>(skip_explicit_ownership))
604 T(std::forward<Args>(args)...);
605 }
606 template <typename T, typename... Args>
607 PROTOBUF_ALWAYS_INLINE T* DoCreateMessage(Args&&... args) {
608 return InternalHelper<T>::Construct(
609 AllocateInternal<T>(InternalHelper<T>::is_destructor_skippable::value),
610 this, std::forward<Args>(args)...);
611 }
612
613 // CreateInArenaStorage is used to implement map field. Without it,
614 // Map need to call generated message's protected arena constructor,
615 // which needs to declare Map as friend of generated message.
616 template <typename T, typename... Args>
617 static void CreateInArenaStorage(T* ptr, Arena* arena, Args&&... args) {
618 CreateInArenaStorageInternal(ptr, arena,
619 typename is_arena_constructable<T>::type(),
620 std::forward<Args>(args)...);
621 RegisterDestructorInternal(
622 ptr, arena,
623 typename InternalHelper<T>::is_destructor_skippable::type());
624 }
625
626 template <typename T, typename... Args>
627 static void CreateInArenaStorageInternal(T* ptr, Arena* arena,
628 std::true_type, Args&&... args) {
629 InternalHelper<T>::Construct(ptr, arena, std::forward<Args>(args)...);
630 }
631 template <typename T, typename... Args>
632 static void CreateInArenaStorageInternal(T* ptr, Arena* /* arena */,
633 std::false_type, Args&&... args) {
634 new (ptr) T(std::forward<Args>(args)...);
635 }
636
637 template <typename T>
638 static void RegisterDestructorInternal(T* /* ptr */, Arena* /* arena */,
639 std::true_type) {}
640 template <typename T>
641 static void RegisterDestructorInternal(T* ptr, Arena* arena,
642 std::false_type) {
643 arena->OwnDestructor(ptr);
644 }
645
646 // These implement Own(), which registers an object for deletion (destructor
647 // call and operator delete()). The second parameter has type 'true_type' if T
648 // is a subtype of Message and 'false_type' otherwise. Collapsing
649 // all template instantiations to one for generic Message reduces code size,
650 // using the virtual destructor instead.
651 template <typename T>
652 PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::true_type) {
653 if (object != NULL) {
654 impl_.AddCleanup(elem: object, cleanup: &internal::arena_delete_object<Message>);
655 }
656 }
657 template <typename T>
658 PROTOBUF_ALWAYS_INLINE void OwnInternal(T* object, std::false_type) {
659 if (object != NULL) {
660 impl_.AddCleanup(elem: object, cleanup: &internal::arena_delete_object<T>);
661 }
662 }
663
664 // Implementation for GetArena(). Only message objects with
665 // InternalArenaConstructable_ tags can be associated with an arena, and such
666 // objects must implement a GetArena() method.
667 template <typename T, typename std::enable_if<
668 is_arena_constructable<T>::value, int>::type = 0>
669 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
670 return InternalHelper<T>::GetArena(value);
671 }
672 template <typename T,
673 typename std::enable_if<!is_arena_constructable<T>::value &&
674 has_get_arena<T>::value,
675 int>::type = 0>
676 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
677 return value->GetArena();
678 }
679 template <typename T,
680 typename std::enable_if<!is_arena_constructable<T>::value &&
681 !has_get_arena<T>::value,
682 int>::type = 0>
683 PROTOBUF_ALWAYS_INLINE static Arena* GetArenaInternal(const T* value) {
684 (void)value;
685 return nullptr;
686 }
687
688 // For friends of arena.
689 void* AllocateAligned(size_t n) {
690 AllocHook(NULL, n);
691 return AllocateAlignedNoHook(n: internal::AlignUpTo8(n));
692 }
693 template<size_t Align>
694 void* AllocateAlignedTo(size_t n) {
695 static_assert(Align > 0, "Alignment must be greater than 0");
696 static_assert((Align & (Align - 1)) == 0, "Alignment must be power of two");
697 if (Align <= 8) return AllocateAligned(n);
698 // TODO(b/151247138): if the pointer would have been aligned already,
699 // this is wasting space. We should pass the alignment down.
700 uintptr_t ptr = reinterpret_cast<uintptr_t>(AllocateAligned(n: n + Align - 8));
701 ptr = (ptr + Align - 1) & -Align;
702 return reinterpret_cast<void*>(ptr);
703 }
704
705 void* AllocateAlignedNoHook(size_t n);
706
707 internal::ArenaImpl impl_;
708
709 void (*on_arena_allocation_)(const std::type_info* allocated_type,
710 uint64 alloc_size, void* cookie);
711 void (*on_arena_reset_)(Arena* arena, void* cookie, uint64 space_used);
712 void (*on_arena_destruction_)(Arena* arena, void* cookie, uint64 space_used);
713
714 // The arena may save a cookie it receives from the external on_init hook
715 // and then use it when calling the on_reset and on_destruction hooks.
716 void* hooks_cookie_;
717
718 template <typename Type>
719 friend class internal::GenericTypeHandler;
720 friend struct internal::ArenaStringPtr; // For AllocateAligned.
721 friend class internal::LazyField; // For CreateMaybeMessage.
722 friend class internal::EpsCopyInputStream; // For parser performance
723 friend class MessageLite;
724 template <typename Key, typename T>
725 friend class Map;
726};
727
728// Defined above for supporting environments without RTTI.
729#undef RTTI_TYPE_ID
730
731} // namespace protobuf
732} // namespace google
733
734#include <google/protobuf/port_undef.inc>
735
736#endif // GOOGLE_PROTOBUF_ARENA_H__
737

source code of include/google/protobuf/arena.h