1/*
2 Copyright (c) 2005-2021 Intel Corporation
3
4 Licensed under the Apache License, Version 2.0 (the "License");
5 you may not use this file except in compliance with the License.
6 You may obtain a copy of the License at
7
8 http://www.apache.org/licenses/LICENSE-2.0
9
10 Unless required by applicable law or agreed to in writing, software
11 distributed under the License is distributed on an "AS IS" BASIS,
12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 See the License for the specific language governing permissions and
14 limitations under the License.
15*/
16
17#ifndef __TBB_cache_aligned_allocator_H
18#define __TBB_cache_aligned_allocator_H
19
20#include "detail/_utils.h"
21#include "detail/_namespace_injection.h"
22#include <cstdlib>
23#include <utility>
24
25#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
26#include <memory_resource>
27#endif
28
29namespace tbb {
30namespace detail {
31
32namespace r1 {
33TBB_EXPORT void* __TBB_EXPORTED_FUNC cache_aligned_allocate(std::size_t size);
34TBB_EXPORT void __TBB_EXPORTED_FUNC cache_aligned_deallocate(void* p);
35TBB_EXPORT std::size_t __TBB_EXPORTED_FUNC cache_line_size();
36}
37
38namespace d1 {
39
40template<typename T>
41class cache_aligned_allocator {
42public:
43 using value_type = T;
44 using propagate_on_container_move_assignment = std::true_type;
45
46 //! Always defined for TBB containers (supported since C++17 for std containers)
47 using is_always_equal = std::true_type;
48
49 cache_aligned_allocator() = default;
50 template<typename U> cache_aligned_allocator(const cache_aligned_allocator<U>&) noexcept {}
51
52 //! Allocate space for n objects, starting on a cache/sector line.
53 __TBB_nodiscard T* allocate(std::size_t n) {
54 return static_cast<T*>(r1::cache_aligned_allocate(size: n * sizeof(value_type)));
55 }
56
57 //! Free block of memory that starts on a cache line
58 void deallocate(T* p, std::size_t) {
59 r1::cache_aligned_deallocate(p);
60 }
61
62 //! Largest value for which method allocate might succeed.
63 std::size_t max_size() const noexcept {
64 return (~std::size_t(0) - r1::cache_line_size()) / sizeof(value_type);
65 }
66
67#if TBB_ALLOCATOR_TRAITS_BROKEN
68 using pointer = value_type*;
69 using const_pointer = const value_type*;
70 using reference = value_type&;
71 using const_reference = const value_type&;
72 using difference_type = std::ptrdiff_t;
73 using size_type = std::size_t;
74 template<typename U> struct rebind {
75 using other = cache_aligned_allocator<U>;
76 };
77 template<typename U, typename... Args>
78 void construct(U *p, Args&&... args)
79 { ::new (p) U(std::forward<Args>(args)...); }
80 void destroy(pointer p) { p->~value_type(); }
81 pointer address(reference x) const { return &x; }
82 const_pointer address(const_reference x) const { return &x; }
83#endif // TBB_ALLOCATOR_TRAITS_BROKEN
84};
85
86#if TBB_ALLOCATOR_TRAITS_BROKEN
87 template<>
88 class cache_aligned_allocator<void> {
89 public:
90 using pointer = void*;
91 using const_pointer = const void*;
92 using value_type = void;
93 template<typename U> struct rebind {
94 using other = cache_aligned_allocator<U>;
95 };
96 };
97#endif
98
99template<typename T, typename U>
100bool operator==(const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>&) noexcept { return true; }
101
102#if !__TBB_CPP20_COMPARISONS_PRESENT
103template<typename T, typename U>
104bool operator!=(const cache_aligned_allocator<T>&, const cache_aligned_allocator<U>&) noexcept { return false; }
105#endif
106
107#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
108
109//! C++17 memory resource wrapper to ensure cache line size alignment
110class cache_aligned_resource : public std::pmr::memory_resource {
111public:
112 cache_aligned_resource() : cache_aligned_resource(std::pmr::get_default_resource()) {}
113 explicit cache_aligned_resource(std::pmr::memory_resource* upstream) : m_upstream(upstream) {}
114
115 std::pmr::memory_resource* upstream_resource() const {
116 return m_upstream;
117 }
118
119private:
120 //! We don't know what memory resource set. Use padding to guarantee alignment
121 void* do_allocate(std::size_t bytes, std::size_t alignment) override {
122 // TODO: make it common with tbb_allocator.cpp
123 std::size_t cache_line_alignment = correct_alignment(alignment);
124 std::size_t space = correct_size(bytes) + cache_line_alignment;
125 std::uintptr_t base = reinterpret_cast<std::uintptr_t>(m_upstream->allocate(bytes: space));
126 __TBB_ASSERT(base != 0, "Upstream resource returned NULL.");
127
128 // Round up to the next cache line (align the base address)
129 std::uintptr_t result = (base + cache_line_alignment) & ~(cache_line_alignment - 1);
130 __TBB_ASSERT((result - base) >= sizeof(std::uintptr_t), "Can`t store a base pointer to the header");
131 __TBB_ASSERT(space - (result - base) >= bytes, "Not enough space for the storage");
132
133 // Record where block actually starts.
134 (reinterpret_cast<std::uintptr_t*>(result))[-1] = base;
135 return reinterpret_cast<void*>(result);
136 }
137
138 void do_deallocate(void* ptr, std::size_t bytes, std::size_t alignment) override {
139 if (ptr) {
140 // Recover where block actually starts
141 std::uintptr_t base = (reinterpret_cast<std::uintptr_t*>(ptr))[-1];
142 m_upstream->deallocate(p: reinterpret_cast<void*>(base), bytes: correct_size(bytes) + correct_alignment(alignment));
143 }
144 }
145
146 bool do_is_equal(const std::pmr::memory_resource& other) const noexcept override {
147 if (this == &other) { return true; }
148#if __TBB_USE_OPTIONAL_RTTI
149 const cache_aligned_resource* other_res = dynamic_cast<const cache_aligned_resource*>(&other);
150 return other_res && (upstream_resource() == other_res->upstream_resource());
151#else
152 return false;
153#endif
154 }
155
156 std::size_t correct_alignment(std::size_t alignment) {
157 __TBB_ASSERT(tbb::detail::is_power_of_two(alignment), "Alignment is not a power of 2");
158#if __TBB_CPP17_HW_INTERFERENCE_SIZE_PRESENT
159 std::size_t cache_line_size = std::hardware_destructive_interference_size;
160#else
161 std::size_t cache_line_size = r1::cache_line_size();
162#endif
163 return alignment < cache_line_size ? cache_line_size : alignment;
164 }
165
166 std::size_t correct_size(std::size_t bytes) {
167 // To handle the case, when small size requested. There could be not
168 // enough space to store the original pointer.
169 return bytes < sizeof(std::uintptr_t) ? sizeof(std::uintptr_t) : bytes;
170 }
171
172 std::pmr::memory_resource* m_upstream;
173};
174
175#endif // __TBB_CPP17_MEMORY_RESOURCE_PRESENT
176
177} // namespace d1
178} // namespace detail
179
180inline namespace v1 {
181using detail::d1::cache_aligned_allocator;
182#if __TBB_CPP17_MEMORY_RESOURCE_PRESENT
183using detail::d1::cache_aligned_resource;
184#endif
185} // namespace v1
186} // namespace tbb
187
188#endif /* __TBB_cache_aligned_allocator_H */
189
190

source code of include/oneapi/tbb/cache_aligned_allocator.h