1// Copyright 2009-2021 Intel Corporation
2// SPDX-License-Identifier: Apache-2.0
3
4#include "alloc.h"
5#include "intrinsics.h"
6#include "sysinfo.h"
7#include "mutex.h"
8
9////////////////////////////////////////////////////////////////////////////////
10/// All Platforms
11////////////////////////////////////////////////////////////////////////////////
12
13namespace embree
14{
15 void* alignedMalloc(size_t size, size_t align)
16 {
17 if (size == 0)
18 return nullptr;
19
20 assert((align & (align-1)) == 0);
21 void* ptr = _mm_malloc(size: size,align: align);
22
23 if (size != 0 && ptr == nullptr)
24 throw std::bad_alloc();
25
26 return ptr;
27 }
28
29 void alignedFree(void* ptr)
30 {
31 if (ptr)
32 _mm_free(p: ptr);
33 }
34
35 static bool huge_pages_enabled = false;
36 static MutexSys os_init_mutex;
37
38 __forceinline bool isHugePageCandidate(const size_t bytes)
39 {
40 if (!huge_pages_enabled)
41 return false;
42
43 /* use huge pages only when memory overhead is low */
44 const size_t hbytes = (bytes+PAGE_SIZE_2M-1) & ~size_t(PAGE_SIZE_2M-1);
45 return 66*(hbytes-bytes) < bytes; // at most 1.5% overhead
46 }
47}
48
49////////////////////////////////////////////////////////////////////////////////
50/// Windows Platform
51////////////////////////////////////////////////////////////////////////////////
52
53#ifdef _WIN32
54
55#define WIN32_LEAN_AND_MEAN
56#include <windows.h>
57#include <malloc.h>
58
59namespace embree
60{
61 bool win_enable_selockmemoryprivilege (bool verbose)
62 {
63 HANDLE hToken;
64 if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, &hToken)) {
65 if (verbose) std::cout << "WARNING: OpenProcessToken failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
66 return false;
67 }
68
69 TOKEN_PRIVILEGES tp;
70 tp.PrivilegeCount = 1;
71 tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
72
73 if (!LookupPrivilegeValueW(nullptr, L"SeLockMemoryPrivilege", &tp.Privileges[0].Luid)) {
74 if (verbose) std::cout << "WARNING: LookupPrivilegeValue failed while trying to enable SeLockMemoryPrivilege: " << GetLastError() << std::endl;
75 return false;
76 }
77
78 SetLastError(ERROR_SUCCESS);
79 if (!AdjustTokenPrivileges(hToken, FALSE, &tp, sizeof(tp), nullptr, 0)) {
80 if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed while trying to enable SeLockMemoryPrivilege" << std::endl;
81 return false;
82 }
83
84 if (GetLastError() == ERROR_NOT_ALL_ASSIGNED) {
85 if (verbose) std::cout << "WARNING: AdjustTokenPrivileges failed to enable SeLockMemoryPrivilege: Add SeLockMemoryPrivilege for current user and run process in elevated mode (Run as administrator)." << std::endl;
86 return false;
87 }
88
89 return true;
90 }
91
92 bool os_init(bool hugepages, bool verbose)
93 {
94 Lock<MutexSys> lock(os_init_mutex);
95
96 if (!hugepages) {
97 huge_pages_enabled = false;
98 return true;
99 }
100
101 if (GetLargePageMinimum() != PAGE_SIZE_2M) {
102 huge_pages_enabled = false;
103 return false;
104 }
105
106 huge_pages_enabled = true;
107 return true;
108 }
109
110 void* os_malloc(size_t bytes, bool& hugepages)
111 {
112 if (bytes == 0) {
113 hugepages = false;
114 return nullptr;
115 }
116
117 /* try direct huge page allocation first */
118 if (isHugePageCandidate(bytes))
119 {
120 int flags = MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES;
121 char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
122 if (ptr != nullptr) {
123 hugepages = true;
124 return ptr;
125 }
126 }
127
128 /* fall back to 4k pages */
129 int flags = MEM_COMMIT | MEM_RESERVE;
130 char* ptr = (char*) VirtualAlloc(nullptr,bytes,flags,PAGE_READWRITE);
131 if (ptr == nullptr) throw std::bad_alloc();
132 hugepages = false;
133 return ptr;
134 }
135
136 size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages)
137 {
138 if (hugepages) // decommitting huge pages seems not to work under Windows
139 return bytesOld;
140
141 const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
142 bytesNew = (bytesNew+pageSize-1) & ~(pageSize-1);
143 bytesOld = (bytesOld+pageSize-1) & ~(pageSize-1);
144 if (bytesNew >= bytesOld)
145 return bytesOld;
146
147 if (!VirtualFree((char*)ptr+bytesNew,bytesOld-bytesNew,MEM_DECOMMIT))
148 throw std::bad_alloc();
149
150 return bytesNew;
151 }
152
153 void os_free(void* ptr, size_t bytes, bool hugepages)
154 {
155 if (bytes == 0)
156 return;
157
158 if (!VirtualFree(ptr,0,MEM_RELEASE))
159 throw std::bad_alloc();
160 }
161
162 void os_advise(void *ptr, size_t bytes)
163 {
164 }
165}
166
167#endif
168
169////////////////////////////////////////////////////////////////////////////////
170/// Unix Platform
171////////////////////////////////////////////////////////////////////////////////
172
173#if defined(__UNIX__)
174
175#include <sys/mman.h>
176#include <errno.h>
177#include <stdlib.h>
178#include <string.h>
179#include <sstream>
180
181#if defined(__MACOSX__)
182#include <mach/vm_statistics.h>
183#endif
184
185namespace embree
186{
187 bool os_init(bool hugepages, bool verbose)
188 {
189 Lock<MutexSys> lock(os_init_mutex);
190
191 if (!hugepages) {
192 huge_pages_enabled = false;
193 return true;
194 }
195
196#if defined(__LINUX__)
197
198 int hugepagesize = 0;
199
200 std::ifstream file;
201 file.open(s: "/proc/meminfo",mode: std::ios::in);
202 if (!file.is_open()) {
203 if (verbose) std::cout << "WARNING: Could not open /proc/meminfo. Huge page support cannot get enabled!" << std::endl;
204 huge_pages_enabled = false;
205 return false;
206 }
207
208 std::string line;
209 while (getline(is&: file,str&: line))
210 {
211 std::stringstream sline(line);
212 while (!sline.eof() && sline.peek() == ' ') sline.ignore();
213 std::string tag; getline(in&: sline,str&: tag,delim: ' ');
214 while (!sline.eof() && sline.peek() == ' ') sline.ignore();
215 std::string val; getline(in&: sline,str&: val,delim: ' ');
216 while (!sline.eof() && sline.peek() == ' ') sline.ignore();
217 std::string unit; getline(in&: sline,str&: unit,delim: ' ');
218 if (tag == "Hugepagesize:" && unit == "kB") {
219 hugepagesize = std::stoi(str: val)*1024;
220 break;
221 }
222 }
223
224 if (hugepagesize != PAGE_SIZE_2M)
225 {
226 if (verbose) std::cout << "WARNING: Only 2MB huge pages supported. Huge page support cannot get enabled!" << std::endl;
227 huge_pages_enabled = false;
228 return false;
229 }
230#endif
231
232 huge_pages_enabled = true;
233 return true;
234 }
235
236 void* os_malloc(size_t bytes, bool& hugepages)
237 {
238 if (bytes == 0) {
239 hugepages = false;
240 return nullptr;
241 }
242
243 /* try direct huge page allocation first */
244 if (isHugePageCandidate(bytes))
245 {
246#if defined(__MACOSX__)
247 void* ptr = mmap(0, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0);
248 if (ptr != MAP_FAILED) {
249 hugepages = true;
250 return ptr;
251 }
252#elif defined(MAP_HUGETLB)
253 void* ptr = mmap(addr: 0, len: bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_HUGETLB, fd: -1, offset: 0);
254 if (ptr != MAP_FAILED) {
255 hugepages = true;
256 return ptr;
257 }
258#endif
259 }
260
261 /* fallback to 4k pages */
262 void* ptr = (char*) mmap(addr: 0, len: bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, fd: -1, offset: 0);
263 if (ptr == MAP_FAILED) throw std::bad_alloc();
264 hugepages = false;
265
266 /* advise huge page hint for THP */
267 os_advise(ptr,bytes);
268 return ptr;
269 }
270
271 size_t os_shrink(void* ptr, size_t bytesNew, size_t bytesOld, bool hugepages)
272 {
273 const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
274 bytesNew = (bytesNew+pageSize-1) & ~(pageSize-1);
275 bytesOld = (bytesOld+pageSize-1) & ~(pageSize-1);
276 if (bytesNew >= bytesOld)
277 return bytesOld;
278
279 if (munmap(addr: (char*)ptr+bytesNew,len: bytesOld-bytesNew) == -1)
280 throw std::bad_alloc();
281
282 return bytesNew;
283 }
284
285 void os_free(void* ptr, size_t bytes, bool hugepages)
286 {
287 if (bytes == 0)
288 return;
289
290 /* for hugepages we need to also align the size */
291 const size_t pageSize = hugepages ? PAGE_SIZE_2M : PAGE_SIZE_4K;
292 bytes = (bytes+pageSize-1) & ~(pageSize-1);
293 if (munmap(addr: ptr,len: bytes) == -1)
294 throw std::bad_alloc();
295 }
296
297 /* hint for transparent huge pages (THP) */
298 void os_advise(void* pptr, size_t bytes)
299 {
300#if defined(MADV_HUGEPAGE)
301 madvise(addr: pptr,len: bytes,MADV_HUGEPAGE);
302#endif
303 }
304}
305
306#endif
307

source code of qtquick3d/src/3rdparty/embree/common/sys/alloc.cpp