1 | // Copyright (c) 2005, 2007, Google Inc. |
2 | // All rights reserved. |
3 | // |
4 | // Redistribution and use in source and binary forms, with or without |
5 | // modification, are permitted provided that the following conditions are |
6 | // met: |
7 | // |
8 | // * Redistributions of source code must retain the above copyright |
9 | // notice, this list of conditions and the following disclaimer. |
10 | // * Redistributions in binary form must reproduce the above |
11 | // copyright notice, this list of conditions and the following disclaimer |
12 | // in the documentation and/or other materials provided with the |
13 | // distribution. |
14 | // * Neither the name of Google Inc. nor the names of its |
15 | // contributors may be used to endorse or promote products derived from |
16 | // this software without specific prior written permission. |
17 | // |
18 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
19 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
20 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
21 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
22 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
24 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
25 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
26 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
27 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
28 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
29 | |
30 | // --- |
31 | // Author: Sanjay Ghemawat |
32 | |
33 | #include "config.h" |
34 | #include "TCSystemAlloc.h" |
35 | |
36 | #include <algorithm> |
37 | #include <fcntl.h> |
38 | #include "Assertions.h" |
39 | #include "TCSpinLock.h" |
40 | #include "UnusedParam.h" |
41 | #include "VMTags.h" |
42 | |
43 | #if HAVE(STDINT_H) |
44 | #include <stdint.h> |
45 | #elif HAVE(INTTYPES_H) |
46 | #include <inttypes.h> |
47 | #else |
48 | #include <sys/types.h> |
49 | #endif |
50 | |
51 | #if OS(WINDOWS) |
52 | #include "windows.h" |
53 | #else |
54 | #include <errno.h> |
55 | #include <unistd.h> |
56 | #include <sys/mman.h> |
57 | #endif |
58 | |
59 | #ifndef MAP_ANONYMOUS |
60 | #define MAP_ANONYMOUS MAP_ANON |
61 | #endif |
62 | |
63 | using namespace std; |
64 | |
65 | // Structure for discovering alignment |
66 | union MemoryAligner { |
67 | void* p; |
68 | double d; |
69 | size_t s; |
70 | }; |
71 | |
72 | static SpinLock spinlock = SPINLOCK_INITIALIZER; |
73 | |
74 | // Page size is initialized on demand |
75 | static size_t pagesize = 0; |
76 | |
77 | // Configuration parameters. |
78 | // |
79 | // if use_devmem is true, either use_sbrk or use_mmap must also be true. |
80 | // For 2.2 kernels, it looks like the sbrk address space (500MBish) and |
81 | // the mmap address space (1300MBish) are disjoint, so we need both allocators |
82 | // to get as much virtual memory as possible. |
83 | #ifndef WTF_CHANGES |
84 | static bool use_devmem = false; |
85 | #endif |
86 | |
87 | #if HAVE(SBRK) |
88 | static bool use_sbrk = false; |
89 | #endif |
90 | |
91 | #if HAVE(MMAP) |
92 | static bool use_mmap = true; |
93 | #endif |
94 | |
95 | #if HAVE(VIRTUALALLOC) |
96 | static bool use_VirtualAlloc = true; |
97 | #endif |
98 | |
99 | // Flags to keep us from retrying allocators that failed. |
100 | static bool devmem_failure = false; |
101 | static bool sbrk_failure = false; |
102 | static bool mmap_failure = false; |
103 | static bool VirtualAlloc_failure = false; |
104 | |
105 | #ifndef WTF_CHANGES |
106 | DEFINE_int32(malloc_devmem_start, 0, |
107 | "Physical memory starting location in MB for /dev/mem allocation." |
108 | " Setting this to 0 disables /dev/mem allocation" ); |
109 | DEFINE_int32(malloc_devmem_limit, 0, |
110 | "Physical memory limit location in MB for /dev/mem allocation." |
111 | " Setting this to 0 means no limit." ); |
112 | #else |
113 | static const int32_t FLAGS_malloc_devmem_start = 0; |
114 | static const int32_t FLAGS_malloc_devmem_limit = 0; |
115 | #endif |
116 | |
117 | #if HAVE(SBRK) |
118 | |
119 | static void* TrySbrk(size_t size, size_t *actual_size, size_t alignment) { |
120 | size = ((size + alignment - 1) / alignment) * alignment; |
121 | |
122 | // could theoretically return the "extra" bytes here, but this |
123 | // is simple and correct. |
124 | if (actual_size) |
125 | *actual_size = size; |
126 | |
127 | void* result = sbrk(delta: size); |
128 | if (result == reinterpret_cast<void*>(-1)) { |
129 | sbrk_failure = true; |
130 | return NULL; |
131 | } |
132 | |
133 | // Is it aligned? |
134 | uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
135 | if ((ptr & (alignment-1)) == 0) return result; |
136 | |
137 | // Try to get more memory for alignment |
138 | size_t = alignment - (ptr & (alignment-1)); |
139 | void* r2 = sbrk(delta: extra); |
140 | if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) { |
141 | // Contiguous with previous result |
142 | return reinterpret_cast<void*>(ptr + extra); |
143 | } |
144 | |
145 | // Give up and ask for "size + alignment - 1" bytes so |
146 | // that we can find an aligned region within it. |
147 | result = sbrk(delta: size + alignment - 1); |
148 | if (result == reinterpret_cast<void*>(-1)) { |
149 | sbrk_failure = true; |
150 | return NULL; |
151 | } |
152 | ptr = reinterpret_cast<uintptr_t>(result); |
153 | if ((ptr & (alignment-1)) != 0) { |
154 | ptr += alignment - (ptr & (alignment-1)); |
155 | } |
156 | return reinterpret_cast<void*>(ptr); |
157 | } |
158 | |
159 | #endif /* HAVE(SBRK) */ |
160 | |
161 | #if HAVE(MMAP) |
162 | |
163 | static void* TryMmap(size_t size, size_t *actual_size, size_t alignment) { |
164 | // Enforce page alignment |
165 | if (pagesize == 0) pagesize = getpagesize(); |
166 | if (alignment < pagesize) alignment = pagesize; |
167 | size = ((size + alignment - 1) / alignment) * alignment; |
168 | |
169 | // could theoretically return the "extra" bytes here, but this |
170 | // is simple and correct. |
171 | if (actual_size) |
172 | *actual_size = size; |
173 | |
174 | // Ask for extra memory if alignment > pagesize |
175 | size_t = 0; |
176 | if (alignment > pagesize) { |
177 | extra = alignment - pagesize; |
178 | } |
179 | void* result = mmap(NULL, len: size + extra, |
180 | PROT_READ | PROT_WRITE, |
181 | MAP_PRIVATE|MAP_ANONYMOUS, |
182 | VM_TAG_FOR_TCMALLOC_MEMORY, offset: 0); |
183 | if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
184 | mmap_failure = true; |
185 | return NULL; |
186 | } |
187 | |
188 | // Adjust the return memory so it is aligned |
189 | uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
190 | size_t adjust = 0; |
191 | if ((ptr & (alignment - 1)) != 0) { |
192 | adjust = alignment - (ptr & (alignment - 1)); |
193 | } |
194 | |
195 | // Return the unused memory to the system |
196 | if (adjust > 0) { |
197 | munmap(addr: reinterpret_cast<char*>(ptr), len: adjust); |
198 | } |
199 | if (adjust < extra) { |
200 | munmap(addr: reinterpret_cast<char*>(ptr + adjust + size), len: extra - adjust); |
201 | } |
202 | |
203 | ptr += adjust; |
204 | return reinterpret_cast<void*>(ptr); |
205 | } |
206 | |
207 | #endif /* HAVE(MMAP) */ |
208 | |
209 | #if HAVE(VIRTUALALLOC) |
210 | |
211 | static void* TryVirtualAlloc(size_t size, size_t *actual_size, size_t alignment) { |
212 | // Enforce page alignment |
213 | if (pagesize == 0) { |
214 | SYSTEM_INFO system_info; |
215 | GetSystemInfo(&system_info); |
216 | pagesize = system_info.dwPageSize; |
217 | } |
218 | |
219 | if (alignment < pagesize) alignment = pagesize; |
220 | size = ((size + alignment - 1) / alignment) * alignment; |
221 | |
222 | // could theoretically return the "extra" bytes here, but this |
223 | // is simple and correct. |
224 | if (actual_size) |
225 | *actual_size = size; |
226 | |
227 | // Ask for extra memory if alignment > pagesize |
228 | size_t extra = 0; |
229 | if (alignment > pagesize) { |
230 | extra = alignment - pagesize; |
231 | } |
232 | void* result = VirtualAlloc(NULL, size + extra, |
233 | MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN, |
234 | PAGE_READWRITE); |
235 | |
236 | if (result == NULL) { |
237 | VirtualAlloc_failure = true; |
238 | return NULL; |
239 | } |
240 | |
241 | // Adjust the return memory so it is aligned |
242 | uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
243 | size_t adjust = 0; |
244 | if ((ptr & (alignment - 1)) != 0) { |
245 | adjust = alignment - (ptr & (alignment - 1)); |
246 | } |
247 | |
248 | // Return the unused memory to the system - we'd like to release but the best we can do |
249 | // is decommit, since Windows only lets you free the whole allocation. |
250 | if (adjust > 0) { |
251 | VirtualFree(reinterpret_cast<void*>(ptr), adjust, MEM_DECOMMIT); |
252 | } |
253 | if (adjust < extra) { |
254 | VirtualFree(reinterpret_cast<void*>(ptr + adjust + size), extra-adjust, MEM_DECOMMIT); |
255 | } |
256 | |
257 | ptr += adjust; |
258 | return reinterpret_cast<void*>(ptr); |
259 | } |
260 | |
261 | #endif /* HAVE(MMAP) */ |
262 | |
263 | #ifndef WTF_CHANGES |
264 | static void* TryDevMem(size_t size, size_t *actual_size, size_t alignment) { |
265 | static bool initialized = false; |
266 | static off_t physmem_base; // next physical memory address to allocate |
267 | static off_t physmem_limit; // maximum physical address allowed |
268 | static int physmem_fd; // file descriptor for /dev/mem |
269 | |
270 | // Check if we should use /dev/mem allocation. Note that it may take |
271 | // a while to get this flag initialized, so meanwhile we fall back to |
272 | // the next allocator. (It looks like 7MB gets allocated before |
273 | // this flag gets initialized -khr.) |
274 | if (FLAGS_malloc_devmem_start == 0) { |
275 | // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to |
276 | // try us again next time. |
277 | return NULL; |
278 | } |
279 | |
280 | if (!initialized) { |
281 | physmem_fd = open("/dev/mem" , O_RDWR); |
282 | if (physmem_fd < 0) { |
283 | devmem_failure = true; |
284 | return NULL; |
285 | } |
286 | physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL; |
287 | physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL; |
288 | initialized = true; |
289 | } |
290 | |
291 | // Enforce page alignment |
292 | if (pagesize == 0) pagesize = getpagesize(); |
293 | if (alignment < pagesize) alignment = pagesize; |
294 | size = ((size + alignment - 1) / alignment) * alignment; |
295 | |
296 | // could theoretically return the "extra" bytes here, but this |
297 | // is simple and correct. |
298 | if (actual_size) |
299 | *actual_size = size; |
300 | |
301 | // Ask for extra memory if alignment > pagesize |
302 | size_t extra = 0; |
303 | if (alignment > pagesize) { |
304 | extra = alignment - pagesize; |
305 | } |
306 | |
307 | // check to see if we have any memory left |
308 | if (physmem_limit != 0 && physmem_base + size + extra > physmem_limit) { |
309 | devmem_failure = true; |
310 | return NULL; |
311 | } |
312 | void *result = mmap(0, size + extra, PROT_READ | PROT_WRITE, |
313 | MAP_SHARED, physmem_fd, physmem_base); |
314 | if (result == reinterpret_cast<void*>(MAP_FAILED)) { |
315 | devmem_failure = true; |
316 | return NULL; |
317 | } |
318 | uintptr_t ptr = reinterpret_cast<uintptr_t>(result); |
319 | |
320 | // Adjust the return memory so it is aligned |
321 | size_t adjust = 0; |
322 | if ((ptr & (alignment - 1)) != 0) { |
323 | adjust = alignment - (ptr & (alignment - 1)); |
324 | } |
325 | |
326 | // Return the unused virtual memory to the system |
327 | if (adjust > 0) { |
328 | munmap(reinterpret_cast<void*>(ptr), adjust); |
329 | } |
330 | if (adjust < extra) { |
331 | munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust); |
332 | } |
333 | |
334 | ptr += adjust; |
335 | physmem_base += adjust + size; |
336 | |
337 | return reinterpret_cast<void*>(ptr); |
338 | } |
339 | #endif |
340 | |
341 | void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, size_t alignment) { |
342 | // Discard requests that overflow |
343 | if (size + alignment < size) return NULL; |
344 | |
345 | SpinLockHolder lock_holder(&spinlock); |
346 | |
347 | // Enforce minimum alignment |
348 | if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner); |
349 | |
350 | // Try twice, once avoiding allocators that failed before, and once |
351 | // more trying all allocators even if they failed before. |
352 | for (int i = 0; i < 2; i++) { |
353 | |
354 | #ifndef WTF_CHANGES |
355 | if (use_devmem && !devmem_failure) { |
356 | void* result = TryDevMem(size, actual_size, alignment); |
357 | if (result != NULL) return result; |
358 | } |
359 | #endif |
360 | |
361 | #if HAVE(SBRK) |
362 | if (use_sbrk && !sbrk_failure) { |
363 | void* result = TrySbrk(size, actual_size, alignment); |
364 | if (result != NULL) return result; |
365 | } |
366 | #endif |
367 | |
368 | #if HAVE(MMAP) |
369 | if (use_mmap && !mmap_failure) { |
370 | void* result = TryMmap(size, actual_size, alignment); |
371 | if (result != NULL) return result; |
372 | } |
373 | #endif |
374 | |
375 | #if HAVE(VIRTUALALLOC) |
376 | if (use_VirtualAlloc && !VirtualAlloc_failure) { |
377 | void* result = TryVirtualAlloc(size, actual_size, alignment); |
378 | if (result != NULL) return result; |
379 | } |
380 | #endif |
381 | |
382 | // nothing worked - reset failure flags and try again |
383 | devmem_failure = false; |
384 | sbrk_failure = false; |
385 | mmap_failure = false; |
386 | VirtualAlloc_failure = false; |
387 | } |
388 | return NULL; |
389 | } |
390 | |
391 | #if HAVE(MADV_FREE_REUSE) |
392 | |
393 | void TCMalloc_SystemRelease(void* start, size_t length) |
394 | { |
395 | while (madvise(start, length, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) { } |
396 | } |
397 | |
398 | #elif HAVE(MADV_FREE) || HAVE(MADV_DONTNEED) |
399 | |
400 | void TCMalloc_SystemRelease(void* start, size_t length) |
401 | { |
402 | // MADV_FREE clears the modified bit on pages, which allows |
403 | // them to be discarded immediately. |
404 | #if HAVE(MADV_FREE) |
405 | const int advice = MADV_FREE; |
406 | #else |
407 | const int advice = MADV_DONTNEED; |
408 | #endif |
409 | if (FLAGS_malloc_devmem_start) { |
410 | // It's not safe to use MADV_DONTNEED if we've been mapping |
411 | // /dev/mem for heap memory |
412 | return; |
413 | } |
414 | if (pagesize == 0) pagesize = getpagesize(); |
415 | const size_t pagemask = pagesize - 1; |
416 | |
417 | size_t new_start = reinterpret_cast<size_t>(start); |
418 | size_t end = new_start + length; |
419 | size_t new_end = end; |
420 | |
421 | // Round up the starting address and round down the ending address |
422 | // to be page aligned: |
423 | new_start = (new_start + pagesize - 1) & ~pagemask; |
424 | new_end = new_end & ~pagemask; |
425 | |
426 | ASSERT((new_start & pagemask) == 0); |
427 | ASSERT((new_end & pagemask) == 0); |
428 | ASSERT(new_start >= reinterpret_cast<size_t>(start)); |
429 | ASSERT(new_end <= end); |
430 | |
431 | if (new_end > new_start) { |
432 | // Note -- ignoring most return codes, because if this fails it |
433 | // doesn't matter... |
434 | while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start, |
435 | advice) == -1 && |
436 | errno == EAGAIN) { |
437 | // NOP |
438 | } |
439 | } |
440 | } |
441 | |
442 | #elif HAVE(MMAP) |
443 | |
444 | void TCMalloc_SystemRelease(void* start, size_t length) |
445 | { |
446 | void* newAddress = mmap(addr: reinterpret_cast<char*>(start), len: length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, fd: -1, offset: 0); |
447 | // If the mmap failed then that's ok, we just won't return the memory to the system. |
448 | ASSERT_UNUSED(newAddress, newAddress == start || newAddress == reinterpret_cast<void*>(MAP_FAILED)); |
449 | } |
450 | |
451 | #elif HAVE(VIRTUALALLOC) |
452 | |
453 | void TCMalloc_SystemRelease(void* start, size_t length) |
454 | { |
455 | if (VirtualFree(start, length, MEM_DECOMMIT)) |
456 | return; |
457 | |
458 | // The decommit may fail if the memory region consists of allocations |
459 | // from more than one call to VirtualAlloc. In this case, fall back to |
460 | // using VirtualQuery to retrieve the allocation boundaries and decommit |
461 | // them each individually. |
462 | |
463 | char* ptr = static_cast<char*>(start); |
464 | char* end = ptr + length; |
465 | MEMORY_BASIC_INFORMATION info; |
466 | while (ptr < end) { |
467 | size_t resultSize = VirtualQuery(ptr, &info, sizeof(info)); |
468 | ASSERT_UNUSED(resultSize, resultSize == sizeof(info)); |
469 | |
470 | size_t decommitSize = min<size_t>(info.RegionSize, end - ptr); |
471 | BOOL success = VirtualFree(ptr, decommitSize, MEM_DECOMMIT); |
472 | ASSERT_UNUSED(success, success); |
473 | ptr += decommitSize; |
474 | } |
475 | } |
476 | |
477 | #else |
478 | |
479 | // Platforms that don't support returning memory use an empty inline version of TCMalloc_SystemRelease |
480 | // declared in TCSystemAlloc.h |
481 | |
482 | #endif |
483 | |
484 | #if HAVE(MADV_FREE_REUSE) |
485 | |
486 | void TCMalloc_SystemCommit(void* start, size_t length) |
487 | { |
488 | while (madvise(start, length, MADV_FREE_REUSE) == -1 && errno == EAGAIN) { } |
489 | } |
490 | |
491 | #elif HAVE(VIRTUALALLOC) |
492 | |
493 | void TCMalloc_SystemCommit(void* start, size_t length) |
494 | { |
495 | if (VirtualAlloc(start, length, MEM_COMMIT, PAGE_READWRITE) == start) |
496 | return; |
497 | |
498 | // The commit may fail if the memory region consists of allocations |
499 | // from more than one call to VirtualAlloc. In this case, fall back to |
500 | // using VirtualQuery to retrieve the allocation boundaries and commit them |
501 | // each individually. |
502 | |
503 | char* ptr = static_cast<char*>(start); |
504 | char* end = ptr + length; |
505 | MEMORY_BASIC_INFORMATION info; |
506 | while (ptr < end) { |
507 | size_t resultSize = VirtualQuery(ptr, &info, sizeof(info)); |
508 | ASSERT_UNUSED(resultSize, resultSize == sizeof(info)); |
509 | |
510 | size_t commitSize = min<size_t>(info.RegionSize, end - ptr); |
511 | void* newAddress = VirtualAlloc(ptr, commitSize, MEM_COMMIT, PAGE_READWRITE); |
512 | ASSERT_UNUSED(newAddress, newAddress == ptr); |
513 | ptr += commitSize; |
514 | } |
515 | } |
516 | |
517 | #else |
518 | |
519 | // Platforms that don't need to explicitly commit memory use an empty inline version of TCMalloc_SystemCommit |
520 | // declared in TCSystemAlloc.h |
521 | |
522 | #endif |
523 | |