1// Copyright (c) 2005, 2007, Google Inc.
2// All rights reserved.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are
6// met:
7//
8// * Redistributions of source code must retain the above copyright
9// notice, this list of conditions and the following disclaimer.
10// * Redistributions in binary form must reproduce the above
11// copyright notice, this list of conditions and the following disclaimer
12// in the documentation and/or other materials provided with the
13// distribution.
14// * Neither the name of Google Inc. nor the names of its
15// contributors may be used to endorse or promote products derived from
16// this software without specific prior written permission.
17//
18// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30// ---
31// Author: Sanjay Ghemawat
32
33#include "config.h"
34#include "TCSystemAlloc.h"
35
36#include <algorithm>
37#include <fcntl.h>
38#include "Assertions.h"
39#include "TCSpinLock.h"
40#include "UnusedParam.h"
41#include "VMTags.h"
42
43#if HAVE(STDINT_H)
44#include <stdint.h>
45#elif HAVE(INTTYPES_H)
46#include <inttypes.h>
47#else
48#include <sys/types.h>
49#endif
50
51#if OS(WINDOWS)
52#include "windows.h"
53#else
54#include <errno.h>
55#include <unistd.h>
56#include <sys/mman.h>
57#endif
58
59#ifndef MAP_ANONYMOUS
60#define MAP_ANONYMOUS MAP_ANON
61#endif
62
63using namespace std;
64
65// Structure for discovering alignment
66union MemoryAligner {
67 void* p;
68 double d;
69 size_t s;
70};
71
72static SpinLock spinlock = SPINLOCK_INITIALIZER;
73
74// Page size is initialized on demand
75static size_t pagesize = 0;
76
77// Configuration parameters.
78//
79// if use_devmem is true, either use_sbrk or use_mmap must also be true.
80// For 2.2 kernels, it looks like the sbrk address space (500MBish) and
81// the mmap address space (1300MBish) are disjoint, so we need both allocators
82// to get as much virtual memory as possible.
83#ifndef WTF_CHANGES
84static bool use_devmem = false;
85#endif
86
87#if HAVE(SBRK)
88static bool use_sbrk = false;
89#endif
90
91#if HAVE(MMAP)
92static bool use_mmap = true;
93#endif
94
95#if HAVE(VIRTUALALLOC)
96static bool use_VirtualAlloc = true;
97#endif
98
99// Flags to keep us from retrying allocators that failed.
100static bool devmem_failure = false;
101static bool sbrk_failure = false;
102static bool mmap_failure = false;
103static bool VirtualAlloc_failure = false;
104
105#ifndef WTF_CHANGES
106DEFINE_int32(malloc_devmem_start, 0,
107 "Physical memory starting location in MB for /dev/mem allocation."
108 " Setting this to 0 disables /dev/mem allocation");
109DEFINE_int32(malloc_devmem_limit, 0,
110 "Physical memory limit location in MB for /dev/mem allocation."
111 " Setting this to 0 means no limit.");
112#else
113static const int32_t FLAGS_malloc_devmem_start = 0;
114static const int32_t FLAGS_malloc_devmem_limit = 0;
115#endif
116
117#if HAVE(SBRK)
118
119static void* TrySbrk(size_t size, size_t *actual_size, size_t alignment) {
120 size = ((size + alignment - 1) / alignment) * alignment;
121
122 // could theoretically return the "extra" bytes here, but this
123 // is simple and correct.
124 if (actual_size)
125 *actual_size = size;
126
127 void* result = sbrk(delta: size);
128 if (result == reinterpret_cast<void*>(-1)) {
129 sbrk_failure = true;
130 return NULL;
131 }
132
133 // Is it aligned?
134 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
135 if ((ptr & (alignment-1)) == 0) return result;
136
137 // Try to get more memory for alignment
138 size_t extra = alignment - (ptr & (alignment-1));
139 void* r2 = sbrk(delta: extra);
140 if (reinterpret_cast<uintptr_t>(r2) == (ptr + size)) {
141 // Contiguous with previous result
142 return reinterpret_cast<void*>(ptr + extra);
143 }
144
145 // Give up and ask for "size + alignment - 1" bytes so
146 // that we can find an aligned region within it.
147 result = sbrk(delta: size + alignment - 1);
148 if (result == reinterpret_cast<void*>(-1)) {
149 sbrk_failure = true;
150 return NULL;
151 }
152 ptr = reinterpret_cast<uintptr_t>(result);
153 if ((ptr & (alignment-1)) != 0) {
154 ptr += alignment - (ptr & (alignment-1));
155 }
156 return reinterpret_cast<void*>(ptr);
157}
158
159#endif /* HAVE(SBRK) */
160
161#if HAVE(MMAP)
162
163static void* TryMmap(size_t size, size_t *actual_size, size_t alignment) {
164 // Enforce page alignment
165 if (pagesize == 0) pagesize = getpagesize();
166 if (alignment < pagesize) alignment = pagesize;
167 size = ((size + alignment - 1) / alignment) * alignment;
168
169 // could theoretically return the "extra" bytes here, but this
170 // is simple and correct.
171 if (actual_size)
172 *actual_size = size;
173
174 // Ask for extra memory if alignment > pagesize
175 size_t extra = 0;
176 if (alignment > pagesize) {
177 extra = alignment - pagesize;
178 }
179 void* result = mmap(NULL, len: size + extra,
180 PROT_READ | PROT_WRITE,
181 MAP_PRIVATE|MAP_ANONYMOUS,
182 VM_TAG_FOR_TCMALLOC_MEMORY, offset: 0);
183 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
184 mmap_failure = true;
185 return NULL;
186 }
187
188 // Adjust the return memory so it is aligned
189 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
190 size_t adjust = 0;
191 if ((ptr & (alignment - 1)) != 0) {
192 adjust = alignment - (ptr & (alignment - 1));
193 }
194
195 // Return the unused memory to the system
196 if (adjust > 0) {
197 munmap(addr: reinterpret_cast<char*>(ptr), len: adjust);
198 }
199 if (adjust < extra) {
200 munmap(addr: reinterpret_cast<char*>(ptr + adjust + size), len: extra - adjust);
201 }
202
203 ptr += adjust;
204 return reinterpret_cast<void*>(ptr);
205}
206
207#endif /* HAVE(MMAP) */
208
209#if HAVE(VIRTUALALLOC)
210
211static void* TryVirtualAlloc(size_t size, size_t *actual_size, size_t alignment) {
212 // Enforce page alignment
213 if (pagesize == 0) {
214 SYSTEM_INFO system_info;
215 GetSystemInfo(&system_info);
216 pagesize = system_info.dwPageSize;
217 }
218
219 if (alignment < pagesize) alignment = pagesize;
220 size = ((size + alignment - 1) / alignment) * alignment;
221
222 // could theoretically return the "extra" bytes here, but this
223 // is simple and correct.
224 if (actual_size)
225 *actual_size = size;
226
227 // Ask for extra memory if alignment > pagesize
228 size_t extra = 0;
229 if (alignment > pagesize) {
230 extra = alignment - pagesize;
231 }
232 void* result = VirtualAlloc(NULL, size + extra,
233 MEM_RESERVE | MEM_COMMIT | MEM_TOP_DOWN,
234 PAGE_READWRITE);
235
236 if (result == NULL) {
237 VirtualAlloc_failure = true;
238 return NULL;
239 }
240
241 // Adjust the return memory so it is aligned
242 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
243 size_t adjust = 0;
244 if ((ptr & (alignment - 1)) != 0) {
245 adjust = alignment - (ptr & (alignment - 1));
246 }
247
248 // Return the unused memory to the system - we'd like to release but the best we can do
249 // is decommit, since Windows only lets you free the whole allocation.
250 if (adjust > 0) {
251 VirtualFree(reinterpret_cast<void*>(ptr), adjust, MEM_DECOMMIT);
252 }
253 if (adjust < extra) {
254 VirtualFree(reinterpret_cast<void*>(ptr + adjust + size), extra-adjust, MEM_DECOMMIT);
255 }
256
257 ptr += adjust;
258 return reinterpret_cast<void*>(ptr);
259}
260
261#endif /* HAVE(MMAP) */
262
263#ifndef WTF_CHANGES
264static void* TryDevMem(size_t size, size_t *actual_size, size_t alignment) {
265 static bool initialized = false;
266 static off_t physmem_base; // next physical memory address to allocate
267 static off_t physmem_limit; // maximum physical address allowed
268 static int physmem_fd; // file descriptor for /dev/mem
269
270 // Check if we should use /dev/mem allocation. Note that it may take
271 // a while to get this flag initialized, so meanwhile we fall back to
272 // the next allocator. (It looks like 7MB gets allocated before
273 // this flag gets initialized -khr.)
274 if (FLAGS_malloc_devmem_start == 0) {
275 // NOTE: not a devmem_failure - we'd like TCMalloc_SystemAlloc to
276 // try us again next time.
277 return NULL;
278 }
279
280 if (!initialized) {
281 physmem_fd = open("/dev/mem", O_RDWR);
282 if (physmem_fd < 0) {
283 devmem_failure = true;
284 return NULL;
285 }
286 physmem_base = FLAGS_malloc_devmem_start*1024LL*1024LL;
287 physmem_limit = FLAGS_malloc_devmem_limit*1024LL*1024LL;
288 initialized = true;
289 }
290
291 // Enforce page alignment
292 if (pagesize == 0) pagesize = getpagesize();
293 if (alignment < pagesize) alignment = pagesize;
294 size = ((size + alignment - 1) / alignment) * alignment;
295
296 // could theoretically return the "extra" bytes here, but this
297 // is simple and correct.
298 if (actual_size)
299 *actual_size = size;
300
301 // Ask for extra memory if alignment > pagesize
302 size_t extra = 0;
303 if (alignment > pagesize) {
304 extra = alignment - pagesize;
305 }
306
307 // check to see if we have any memory left
308 if (physmem_limit != 0 && physmem_base + size + extra > physmem_limit) {
309 devmem_failure = true;
310 return NULL;
311 }
312 void *result = mmap(0, size + extra, PROT_READ | PROT_WRITE,
313 MAP_SHARED, physmem_fd, physmem_base);
314 if (result == reinterpret_cast<void*>(MAP_FAILED)) {
315 devmem_failure = true;
316 return NULL;
317 }
318 uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
319
320 // Adjust the return memory so it is aligned
321 size_t adjust = 0;
322 if ((ptr & (alignment - 1)) != 0) {
323 adjust = alignment - (ptr & (alignment - 1));
324 }
325
326 // Return the unused virtual memory to the system
327 if (adjust > 0) {
328 munmap(reinterpret_cast<void*>(ptr), adjust);
329 }
330 if (adjust < extra) {
331 munmap(reinterpret_cast<void*>(ptr + adjust + size), extra - adjust);
332 }
333
334 ptr += adjust;
335 physmem_base += adjust + size;
336
337 return reinterpret_cast<void*>(ptr);
338}
339#endif
340
341void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size, size_t alignment) {
342 // Discard requests that overflow
343 if (size + alignment < size) return NULL;
344
345 SpinLockHolder lock_holder(&spinlock);
346
347 // Enforce minimum alignment
348 if (alignment < sizeof(MemoryAligner)) alignment = sizeof(MemoryAligner);
349
350 // Try twice, once avoiding allocators that failed before, and once
351 // more trying all allocators even if they failed before.
352 for (int i = 0; i < 2; i++) {
353
354#ifndef WTF_CHANGES
355 if (use_devmem && !devmem_failure) {
356 void* result = TryDevMem(size, actual_size, alignment);
357 if (result != NULL) return result;
358 }
359#endif
360
361#if HAVE(SBRK)
362 if (use_sbrk && !sbrk_failure) {
363 void* result = TrySbrk(size, actual_size, alignment);
364 if (result != NULL) return result;
365 }
366#endif
367
368#if HAVE(MMAP)
369 if (use_mmap && !mmap_failure) {
370 void* result = TryMmap(size, actual_size, alignment);
371 if (result != NULL) return result;
372 }
373#endif
374
375#if HAVE(VIRTUALALLOC)
376 if (use_VirtualAlloc && !VirtualAlloc_failure) {
377 void* result = TryVirtualAlloc(size, actual_size, alignment);
378 if (result != NULL) return result;
379 }
380#endif
381
382 // nothing worked - reset failure flags and try again
383 devmem_failure = false;
384 sbrk_failure = false;
385 mmap_failure = false;
386 VirtualAlloc_failure = false;
387 }
388 return NULL;
389}
390
391#if HAVE(MADV_FREE_REUSE)
392
393void TCMalloc_SystemRelease(void* start, size_t length)
394{
395 while (madvise(start, length, MADV_FREE_REUSABLE) == -1 && errno == EAGAIN) { }
396}
397
398#elif HAVE(MADV_FREE) || HAVE(MADV_DONTNEED)
399
400void TCMalloc_SystemRelease(void* start, size_t length)
401{
402 // MADV_FREE clears the modified bit on pages, which allows
403 // them to be discarded immediately.
404#if HAVE(MADV_FREE)
405 const int advice = MADV_FREE;
406#else
407 const int advice = MADV_DONTNEED;
408#endif
409 if (FLAGS_malloc_devmem_start) {
410 // It's not safe to use MADV_DONTNEED if we've been mapping
411 // /dev/mem for heap memory
412 return;
413 }
414 if (pagesize == 0) pagesize = getpagesize();
415 const size_t pagemask = pagesize - 1;
416
417 size_t new_start = reinterpret_cast<size_t>(start);
418 size_t end = new_start + length;
419 size_t new_end = end;
420
421 // Round up the starting address and round down the ending address
422 // to be page aligned:
423 new_start = (new_start + pagesize - 1) & ~pagemask;
424 new_end = new_end & ~pagemask;
425
426 ASSERT((new_start & pagemask) == 0);
427 ASSERT((new_end & pagemask) == 0);
428 ASSERT(new_start >= reinterpret_cast<size_t>(start));
429 ASSERT(new_end <= end);
430
431 if (new_end > new_start) {
432 // Note -- ignoring most return codes, because if this fails it
433 // doesn't matter...
434 while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
435 advice) == -1 &&
436 errno == EAGAIN) {
437 // NOP
438 }
439 }
440}
441
442#elif HAVE(MMAP)
443
444void TCMalloc_SystemRelease(void* start, size_t length)
445{
446 void* newAddress = mmap(addr: reinterpret_cast<char*>(start), len: length, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, fd: -1, offset: 0);
447 // If the mmap failed then that's ok, we just won't return the memory to the system.
448 ASSERT_UNUSED(newAddress, newAddress == start || newAddress == reinterpret_cast<void*>(MAP_FAILED));
449}
450
451#elif HAVE(VIRTUALALLOC)
452
453void TCMalloc_SystemRelease(void* start, size_t length)
454{
455 if (VirtualFree(start, length, MEM_DECOMMIT))
456 return;
457
458 // The decommit may fail if the memory region consists of allocations
459 // from more than one call to VirtualAlloc. In this case, fall back to
460 // using VirtualQuery to retrieve the allocation boundaries and decommit
461 // them each individually.
462
463 char* ptr = static_cast<char*>(start);
464 char* end = ptr + length;
465 MEMORY_BASIC_INFORMATION info;
466 while (ptr < end) {
467 size_t resultSize = VirtualQuery(ptr, &info, sizeof(info));
468 ASSERT_UNUSED(resultSize, resultSize == sizeof(info));
469
470 size_t decommitSize = min<size_t>(info.RegionSize, end - ptr);
471 BOOL success = VirtualFree(ptr, decommitSize, MEM_DECOMMIT);
472 ASSERT_UNUSED(success, success);
473 ptr += decommitSize;
474 }
475}
476
477#else
478
479// Platforms that don't support returning memory use an empty inline version of TCMalloc_SystemRelease
480// declared in TCSystemAlloc.h
481
482#endif
483
484#if HAVE(MADV_FREE_REUSE)
485
486void TCMalloc_SystemCommit(void* start, size_t length)
487{
488 while (madvise(start, length, MADV_FREE_REUSE) == -1 && errno == EAGAIN) { }
489}
490
491#elif HAVE(VIRTUALALLOC)
492
493void TCMalloc_SystemCommit(void* start, size_t length)
494{
495 if (VirtualAlloc(start, length, MEM_COMMIT, PAGE_READWRITE) == start)
496 return;
497
498 // The commit may fail if the memory region consists of allocations
499 // from more than one call to VirtualAlloc. In this case, fall back to
500 // using VirtualQuery to retrieve the allocation boundaries and commit them
501 // each individually.
502
503 char* ptr = static_cast<char*>(start);
504 char* end = ptr + length;
505 MEMORY_BASIC_INFORMATION info;
506 while (ptr < end) {
507 size_t resultSize = VirtualQuery(ptr, &info, sizeof(info));
508 ASSERT_UNUSED(resultSize, resultSize == sizeof(info));
509
510 size_t commitSize = min<size_t>(info.RegionSize, end - ptr);
511 void* newAddress = VirtualAlloc(ptr, commitSize, MEM_COMMIT, PAGE_READWRITE);
512 ASSERT_UNUSED(newAddress, newAddress == ptr);
513 ptr += commitSize;
514 }
515}
516
517#else
518
519// Platforms that don't need to explicitly commit memory use an empty inline version of TCMalloc_SystemCommit
520// declared in TCSystemAlloc.h
521
522#endif
523

source code of qtscript/src/3rdparty/javascriptcore/JavaScriptCore/wtf/TCSystemAlloc.cpp