1/* Malloc implementation for multiple threads without lock contention.
2 Copyright (C) 2001-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public License as
7 published by the Free Software Foundation; either version 2.1 of the
8 License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; see the file COPYING.LIB. If
17 not, see <https://www.gnu.org/licenses/>. */
18
19#include <stdbool.h>
20#include <setvmaname.h>
21
22#define TUNABLE_NAMESPACE malloc
23#include <elf/dl-tunables.h>
24
25/* Compile-time constants. */
26
27#define HEAP_MIN_SIZE (32 * 1024)
28#ifndef HEAP_MAX_SIZE
29# ifdef DEFAULT_MMAP_THRESHOLD_MAX
30# define HEAP_MAX_SIZE (2 * DEFAULT_MMAP_THRESHOLD_MAX)
31# else
32# define HEAP_MAX_SIZE (1024 * 1024) /* must be a power of two */
33# endif
34#endif
35
36/* HEAP_MIN_SIZE and HEAP_MAX_SIZE limit the size of mmap()ed heaps
37 that are dynamically created for multi-threaded programs. The
38 maximum size must be a power of two, for fast determination of
39 which heap belongs to a chunk. It should be much larger than the
40 mmap threshold, so that requests with a size just below that
41 threshold can be fulfilled without creating too many heaps. */
42
43/* When huge pages are used to create new arenas, the maximum and minimum
44 size are based on the runtime defined huge page size. */
45
46static inline size_t
47heap_min_size (void)
48{
49 return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize;
50}
51
52static inline size_t
53heap_max_size (void)
54{
55 return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4;
56}
57
58/***************************************************************************/
59
60#define top(ar_ptr) ((ar_ptr)->top)
61
62/* A heap is a single contiguous memory region holding (coalesceable)
63 malloc_chunks. It is allocated with mmap() and always starts at an
64 address aligned to HEAP_MAX_SIZE. */
65
66typedef struct _heap_info
67{
68 mstate ar_ptr; /* Arena for this heap. */
69 struct _heap_info *prev; /* Previous heap. */
70 size_t size; /* Current size in bytes. */
71 size_t mprotect_size; /* Size in bytes that has been mprotected
72 PROT_READ|PROT_WRITE. */
73 size_t pagesize; /* Page size used when allocating the arena. */
74 /* Make sure the following data is properly aligned, particularly
75 that sizeof (heap_info) + 2 * SIZE_SZ is a multiple of
76 MALLOC_ALIGNMENT. */
77 char pad[-3 * SIZE_SZ & MALLOC_ALIGN_MASK];
78} heap_info;
79
80/* Get a compile-time error if the heap_info padding is not correct
81 to make alignment work as expected in sYSMALLOc. */
82extern int sanity_check_heap_info_alignment[(sizeof (heap_info)
83 + 2 * SIZE_SZ) % MALLOC_ALIGNMENT
84 ? -1 : 1];
85
86/* Thread specific data. */
87
88static __thread mstate thread_arena attribute_tls_model_ie;
89
90/* Arena free list. free_list_lock synchronizes access to the
91 free_list variable below, and the next_free and attached_threads
92 members of struct malloc_state objects. No other locks must be
93 acquired after free_list_lock has been acquired. */
94
95__libc_lock_define_initialized (static, free_list_lock);
96#if IS_IN (libc)
97static size_t narenas = 1;
98#endif
99static mstate free_list;
100
101/* list_lock prevents concurrent writes to the next member of struct
102 malloc_state objects.
103
104 Read access to the next member is supposed to synchronize with the
105 atomic_write_barrier and the write to the next member in
106 _int_new_arena. This suffers from data races; see the FIXME
107 comments in _int_new_arena and reused_arena.
108
109 list_lock also prevents concurrent forks. At the time list_lock is
110 acquired, no arena lock must have been acquired, but it is
111 permitted to acquire arena locks subsequently, while list_lock is
112 acquired. */
113__libc_lock_define_initialized (static, list_lock);
114
115/* Already initialized? */
116static bool __malloc_initialized = false;
117
118/**************************************************************************/
119
120
121/* arena_get() acquires an arena and locks the corresponding mutex.
122 First, try the one last locked successfully by this thread. (This
123 is the common case and handled with a macro for speed.) Then, loop
124 once over the circularly linked list of arenas. If no arena is
125 readily available, create a new one. In this latter case, `size'
126 is just a hint as to how much memory will be required immediately
127 in the new arena. */
128
129#define arena_get(ptr, size) do { \
130 ptr = thread_arena; \
131 arena_lock (ptr, size); \
132 } while (0)
133
134#define arena_lock(ptr, size) do { \
135 if (ptr) \
136 __libc_lock_lock (ptr->mutex); \
137 else \
138 ptr = arena_get2 ((size), NULL); \
139 } while (0)
140
141/* find the heap and corresponding arena for a given ptr */
142
143static inline heap_info *
144heap_for_ptr (void *ptr)
145{
146 size_t max_size = heap_max_size ();
147 return PTR_ALIGN_DOWN (ptr, max_size);
148}
149
150static inline struct malloc_state *
151arena_for_chunk (mchunkptr ptr)
152{
153 return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr;
154}
155
156
157/**************************************************************************/
158
159/* atfork support. */
160
161/* The following three functions are called around fork from a
162 multi-threaded process. We do not use the general fork handler
163 mechanism to make sure that our handlers are the last ones being
164 called, so that other fork handlers can use the malloc
165 subsystem. */
166
167void
168__malloc_fork_lock_parent (void)
169{
170 if (!__malloc_initialized)
171 return;
172
173 /* We do not acquire free_list_lock here because we completely
174 reconstruct free_list in __malloc_fork_unlock_child. */
175
176 __libc_lock_lock (list_lock);
177
178 for (mstate ar_ptr = &main_arena;; )
179 {
180 __libc_lock_lock (ar_ptr->mutex);
181 ar_ptr = ar_ptr->next;
182 if (ar_ptr == &main_arena)
183 break;
184 }
185}
186
187void
188__malloc_fork_unlock_parent (void)
189{
190 if (!__malloc_initialized)
191 return;
192
193 for (mstate ar_ptr = &main_arena;; )
194 {
195 __libc_lock_unlock (ar_ptr->mutex);
196 ar_ptr = ar_ptr->next;
197 if (ar_ptr == &main_arena)
198 break;
199 }
200 __libc_lock_unlock (list_lock);
201}
202
203void
204__malloc_fork_unlock_child (void)
205{
206 if (!__malloc_initialized)
207 return;
208
209 /* Push all arenas to the free list, except thread_arena, which is
210 attached to the current thread. */
211 __libc_lock_init (free_list_lock);
212 if (thread_arena != NULL)
213 thread_arena->attached_threads = 1;
214 free_list = NULL;
215 for (mstate ar_ptr = &main_arena;; )
216 {
217 __libc_lock_init (ar_ptr->mutex);
218 if (ar_ptr != thread_arena)
219 {
220 /* This arena is no longer attached to any thread. */
221 ar_ptr->attached_threads = 0;
222 ar_ptr->next_free = free_list;
223 free_list = ar_ptr;
224 }
225 ar_ptr = ar_ptr->next;
226 if (ar_ptr == &main_arena)
227 break;
228 }
229
230 __libc_lock_init (list_lock);
231}
232
233#define TUNABLE_CALLBACK_FNDECL(__name, __type) \
234static inline int do_ ## __name (__type value); \
235static void \
236TUNABLE_CALLBACK (__name) (tunable_val_t *valp) \
237{ \
238 __type value = (__type) (valp)->numval; \
239 do_ ## __name (value); \
240}
241
242TUNABLE_CALLBACK_FNDECL (set_mmap_threshold, size_t)
243TUNABLE_CALLBACK_FNDECL (set_mmaps_max, int32_t)
244TUNABLE_CALLBACK_FNDECL (set_top_pad, size_t)
245TUNABLE_CALLBACK_FNDECL (set_perturb_byte, int32_t)
246TUNABLE_CALLBACK_FNDECL (set_trim_threshold, size_t)
247TUNABLE_CALLBACK_FNDECL (set_arena_max, size_t)
248TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t)
249#if USE_TCACHE
250TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t)
251TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t)
252TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t)
253#endif
254TUNABLE_CALLBACK_FNDECL (set_mxfast, size_t)
255TUNABLE_CALLBACK_FNDECL (set_hugetlb, size_t)
256
257#if USE_TCACHE
258static void tcache_key_initialize (void);
259#endif
260
261static void
262ptmalloc_init (void)
263{
264 if (__malloc_initialized)
265 return;
266
267 __malloc_initialized = true;
268
269#if USE_TCACHE
270 tcache_key_initialize ();
271#endif
272
273#ifdef USE_MTAG
274 if ((TUNABLE_GET_FULL (glibc, mem, tagging, int32_t, NULL) & 1) != 0)
275 {
276 /* If the tunable says that we should be using tagged memory
277 and that morecore does not support tagged regions, then
278 disable it. */
279 if (__MTAG_SBRK_UNTAGGED)
280 __always_fail_morecore = true;
281
282 mtag_enabled = true;
283 mtag_mmap_flags = __MTAG_MMAP_FLAGS;
284 }
285#endif
286
287#if defined SHARED && IS_IN (libc)
288 /* In case this libc copy is in a non-default namespace, never use
289 brk. Likewise if dlopened from statically linked program. The
290 generic sbrk implementation also enforces this, but it is not
291 used on Hurd. */
292 if (!__libc_initial)
293 __always_fail_morecore = true;
294#endif
295
296 thread_arena = &main_arena;
297
298 malloc_init_state (av: &main_arena);
299
300 TUNABLE_GET (top_pad, size_t, TUNABLE_CALLBACK (set_top_pad));
301 TUNABLE_GET (perturb, int32_t, TUNABLE_CALLBACK (set_perturb_byte));
302 TUNABLE_GET (mmap_threshold, size_t, TUNABLE_CALLBACK (set_mmap_threshold));
303 TUNABLE_GET (trim_threshold, size_t, TUNABLE_CALLBACK (set_trim_threshold));
304 TUNABLE_GET (mmap_max, int32_t, TUNABLE_CALLBACK (set_mmaps_max));
305 TUNABLE_GET (arena_max, size_t, TUNABLE_CALLBACK (set_arena_max));
306 TUNABLE_GET (arena_test, size_t, TUNABLE_CALLBACK (set_arena_test));
307# if USE_TCACHE
308 TUNABLE_GET (tcache_max, size_t, TUNABLE_CALLBACK (set_tcache_max));
309 TUNABLE_GET (tcache_count, size_t, TUNABLE_CALLBACK (set_tcache_count));
310 TUNABLE_GET (tcache_unsorted_limit, size_t,
311 TUNABLE_CALLBACK (set_tcache_unsorted_limit));
312# endif
313 TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast));
314 TUNABLE_GET (hugetlb, size_t, TUNABLE_CALLBACK (set_hugetlb));
315
316 if (mp_.hp_pagesize > 0)
317 {
318 /* Force mmap for main arena instead of sbrk, so MAP_HUGETLB is always
319 tried. Also tune the mmap threshold, so allocation smaller than the
320 large page will also try to use large pages by falling back
321 to sysmalloc_mmap_fallback on sysmalloc. */
322 if (!TUNABLE_IS_INITIALIZED (mmap_threshold))
323 do_set_mmap_threshold (value: mp_.hp_pagesize);
324 __always_fail_morecore = true;
325 }
326}
327
328/* Managing heaps and arenas (for concurrent threads) */
329
330#if MALLOC_DEBUG > 1
331
332/* Print the complete contents of a single heap to stderr. */
333
334static void
335dump_heap (heap_info *heap)
336{
337 char *ptr;
338 mchunkptr p;
339
340 fprintf (stderr, "Heap %p, size %10lx:\n", heap, (long) heap->size);
341 ptr = (heap->ar_ptr != (mstate) (heap + 1)) ?
342 (char *) (heap + 1) : (char *) (heap + 1) + sizeof (struct malloc_state);
343 p = (mchunkptr) (((uintptr_t) ptr + MALLOC_ALIGN_MASK) &
344 ~MALLOC_ALIGN_MASK);
345 for (;; )
346 {
347 fprintf (stderr, "chunk %p size %10lx", p, (long) chunksize_nomask(p));
348 if (p == top (heap->ar_ptr))
349 {
350 fprintf (stderr, " (top)\n");
351 break;
352 }
353 else if (chunksize_nomask(p) == (0 | PREV_INUSE))
354 {
355 fprintf (stderr, " (fence)\n");
356 break;
357 }
358 fprintf (stderr, "\n");
359 p = next_chunk (p);
360 }
361}
362#endif /* MALLOC_DEBUG > 1 */
363
364/* If consecutive mmap (0, HEAP_MAX_SIZE << 1, ...) calls return decreasing
365 addresses as opposed to increasing, new_heap would badly fragment the
366 address space. In that case remember the second HEAP_MAX_SIZE part
367 aligned to HEAP_MAX_SIZE from last mmap (0, HEAP_MAX_SIZE << 1, ...)
368 call (if it is already aligned) and try to reuse it next time. We need
369 no locking for it, as kernel ensures the atomicity for us - worst case
370 we'll call mmap (addr, HEAP_MAX_SIZE, ...) for some value of addr in
371 multiple threads, but only one will succeed. */
372static char *aligned_heap_area;
373
374/* Create a new heap. size is automatically rounded up to a multiple
375 of the page size. */
376
377static heap_info *
378alloc_new_heap (size_t size, size_t top_pad, size_t pagesize,
379 int mmap_flags)
380{
381 char *p1, *p2;
382 unsigned long ul;
383 heap_info *h;
384 size_t min_size = heap_min_size ();
385 size_t max_size = heap_max_size ();
386
387 if (size + top_pad < min_size)
388 size = min_size;
389 else if (size + top_pad <= max_size)
390 size += top_pad;
391 else if (size > max_size)
392 return 0;
393 else
394 size = max_size;
395 size = ALIGN_UP (size, pagesize);
396
397 /* A memory region aligned to a multiple of max_size is needed.
398 No swap space needs to be reserved for the following large
399 mapping (on Linux, this is the case for all non-writable mappings
400 anyway). */
401 p2 = MAP_FAILED;
402 if (aligned_heap_area)
403 {
404 p2 = (char *) MMAP (aligned_heap_area, max_size, PROT_NONE, mmap_flags);
405 aligned_heap_area = NULL;
406 if (p2 != MAP_FAILED && ((unsigned long) p2 & (max_size - 1)))
407 {
408 __munmap (addr: p2, len: max_size);
409 p2 = MAP_FAILED;
410 }
411 }
412 if (p2 == MAP_FAILED)
413 {
414 p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags);
415 if (p1 != MAP_FAILED)
416 {
417 p2 = (char *) (((uintptr_t) p1 + (max_size - 1))
418 & ~(max_size - 1));
419 ul = p2 - p1;
420 if (ul)
421 __munmap (addr: p1, len: ul);
422 else
423 aligned_heap_area = p2 + max_size;
424 __munmap (addr: p2 + max_size, len: max_size - ul);
425 }
426 else
427 {
428 /* Try to take the chance that an allocation of only max_size
429 is already aligned. */
430 p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags);
431 if (p2 == MAP_FAILED)
432 return 0;
433
434 if ((unsigned long) p2 & (max_size - 1))
435 {
436 __munmap (addr: p2, len: max_size);
437 return 0;
438 }
439 }
440 }
441 if (__mprotect (addr: p2, len: size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0)
442 {
443 __munmap (addr: p2, len: max_size);
444 return 0;
445 }
446
447 /* Only considere the actual usable range. */
448 __set_vma_name (start: p2, len: size, name: " glibc: malloc arena");
449
450 madvise_thp (p: p2, size);
451
452 h = (heap_info *) p2;
453 h->size = size;
454 h->mprotect_size = size;
455 h->pagesize = pagesize;
456 LIBC_PROBE (memory_heap_new, 2, h, h->size);
457 return h;
458}
459
460static heap_info *
461new_heap (size_t size, size_t top_pad)
462{
463 if (__glibc_unlikely (mp_.hp_pagesize != 0))
464 {
465 heap_info *h = alloc_new_heap (size, top_pad, pagesize: mp_.hp_pagesize,
466 mmap_flags: mp_.hp_flags);
467 if (h != NULL)
468 return h;
469 }
470 return alloc_new_heap (size, top_pad, GLRO (dl_pagesize), mmap_flags: 0);
471}
472
473/* Grow a heap. size is automatically rounded up to a
474 multiple of the page size. */
475
476static int
477grow_heap (heap_info *h, long diff)
478{
479 size_t pagesize = h->pagesize;
480 size_t max_size = heap_max_size ();
481 long new_size;
482
483 diff = ALIGN_UP (diff, pagesize);
484 new_size = (long) h->size + diff;
485 if ((unsigned long) new_size > (unsigned long) max_size)
486 return -1;
487
488 if ((unsigned long) new_size > h->mprotect_size)
489 {
490 if (__mprotect (addr: (char *) h + h->mprotect_size,
491 len: (unsigned long) new_size - h->mprotect_size,
492 mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0)
493 return -2;
494
495 h->mprotect_size = new_size;
496 }
497
498 h->size = new_size;
499 LIBC_PROBE (memory_heap_more, 2, h, h->size);
500 return 0;
501}
502
503/* Shrink a heap. */
504
505static int
506shrink_heap (heap_info *h, long diff)
507{
508 long new_size;
509
510 new_size = (long) h->size - diff;
511 if (new_size < (long) sizeof (*h))
512 return -1;
513
514 /* Try to re-map the extra heap space freshly to save memory, and make it
515 inaccessible. See malloc-sysdep.h to know when this is true. */
516 if (__glibc_unlikely (check_may_shrink_heap ()))
517 {
518 if ((char *) MMAP ((char *) h + new_size, diff, PROT_NONE,
519 MAP_FIXED) == (char *) MAP_FAILED)
520 return -2;
521
522 h->mprotect_size = new_size;
523 }
524 else
525 __madvise (addr: (char *) h + new_size, len: diff, MADV_DONTNEED);
526 /*fprintf(stderr, "shrink %p %08lx\n", h, new_size);*/
527
528 h->size = new_size;
529 LIBC_PROBE (memory_heap_less, 2, h, h->size);
530 return 0;
531}
532
533/* Delete a heap. */
534
535static int
536heap_trim (heap_info *heap, size_t pad)
537{
538 mstate ar_ptr = heap->ar_ptr;
539 mchunkptr top_chunk = top (ar_ptr), p;
540 heap_info *prev_heap;
541 long new_size, top_size, top_area, extra, prev_size, misalign;
542 size_t max_size = heap_max_size ();
543
544 /* Can this heap go away completely? */
545 while (top_chunk == chunk_at_offset (heap, sizeof (*heap)))
546 {
547 prev_heap = heap->prev;
548 prev_size = prev_heap->size - (MINSIZE - 2 * SIZE_SZ);
549 p = chunk_at_offset (prev_heap, prev_size);
550 /* fencepost must be properly aligned. */
551 misalign = ((long) p) & MALLOC_ALIGN_MASK;
552 p = chunk_at_offset (prev_heap, prev_size - misalign);
553 assert (chunksize_nomask (p) == (0 | PREV_INUSE)); /* must be fencepost */
554 p = prev_chunk (p);
555 new_size = chunksize (p) + (MINSIZE - 2 * SIZE_SZ) + misalign;
556 assert (new_size > 0 && new_size < (long) (2 * MINSIZE));
557 if (!prev_inuse (p))
558 new_size += prev_size (p);
559 assert (new_size > 0 && new_size < max_size);
560 if (new_size + (max_size - prev_heap->size) < pad + MINSIZE
561 + heap->pagesize)
562 break;
563 ar_ptr->system_mem -= heap->size;
564 LIBC_PROBE (memory_heap_free, 2, heap, heap->size);
565 if ((char *) heap + max_size == aligned_heap_area)
566 aligned_heap_area = NULL;
567 __munmap (addr: heap, len: max_size);
568 heap = prev_heap;
569 if (!prev_inuse (p)) /* consolidate backward */
570 {
571 p = prev_chunk (p);
572 unlink_chunk (av: ar_ptr, p);
573 }
574 assert (((unsigned long) ((char *) p + new_size) & (heap->pagesize - 1))
575 == 0);
576 assert (((char *) p + new_size) == ((char *) heap + heap->size));
577 top (ar_ptr) = top_chunk = p;
578 set_head (top_chunk, new_size | PREV_INUSE);
579 /*check_chunk(ar_ptr, top_chunk);*/
580 }
581
582 /* Uses similar logic for per-thread arenas as the main arena with systrim
583 and _int_free by preserving the top pad and rounding down to the nearest
584 page. */
585 top_size = chunksize (top_chunk);
586 if ((unsigned long)(top_size) <
587 (unsigned long)(mp_.trim_threshold))
588 return 0;
589
590 top_area = top_size - MINSIZE - 1;
591 if (top_area < 0 || (size_t) top_area <= pad)
592 return 0;
593
594 /* Release in pagesize units and round down to the nearest page. */
595 extra = ALIGN_DOWN(top_area - pad, heap->pagesize);
596 if (extra == 0)
597 return 0;
598
599 /* Try to shrink. */
600 if (shrink_heap (h: heap, diff: extra) != 0)
601 return 0;
602
603 ar_ptr->system_mem -= extra;
604
605 /* Success. Adjust top accordingly. */
606 set_head (top_chunk, (top_size - extra) | PREV_INUSE);
607 /*check_chunk(ar_ptr, top_chunk);*/
608 return 1;
609}
610
611/* Create a new arena with initial size "size". */
612
613#if IS_IN (libc)
614/* If REPLACED_ARENA is not NULL, detach it from this thread. Must be
615 called while free_list_lock is held. */
616static void
617detach_arena (mstate replaced_arena)
618{
619 if (replaced_arena != NULL)
620 {
621 assert (replaced_arena->attached_threads > 0);
622 /* The current implementation only detaches from main_arena in
623 case of allocation failure. This means that it is likely not
624 beneficial to put the arena on free_list even if the
625 reference count reaches zero. */
626 --replaced_arena->attached_threads;
627 }
628}
629
630static mstate
631_int_new_arena (size_t size)
632{
633 mstate a;
634 heap_info *h;
635 char *ptr;
636 unsigned long misalign;
637
638 h = new_heap (size + (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT),
639 mp_.top_pad);
640 if (!h)
641 {
642 /* Maybe size is too large to fit in a single heap. So, just try
643 to create a minimally-sized arena and let _int_malloc() attempt
644 to deal with the large request via mmap_chunk(). */
645 h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad);
646 if (!h)
647 return 0;
648 }
649 a = h->ar_ptr = (mstate) (h + 1);
650 malloc_init_state (a);
651 a->attached_threads = 1;
652 /*a->next = NULL;*/
653 a->system_mem = a->max_system_mem = h->size;
654
655 /* Set up the top chunk, with proper alignment. */
656 ptr = (char *) (a + 1);
657 misalign = (uintptr_t) chunk2mem (ptr) & MALLOC_ALIGN_MASK;
658 if (misalign > 0)
659 ptr += MALLOC_ALIGNMENT - misalign;
660 top (a) = (mchunkptr) ptr;
661 set_head (top (a), (((char *) h + h->size) - ptr) | PREV_INUSE);
662
663 LIBC_PROBE (memory_arena_new, 2, a, size);
664 mstate replaced_arena = thread_arena;
665 thread_arena = a;
666 __libc_lock_init (a->mutex);
667
668 __libc_lock_lock (list_lock);
669
670 /* Add the new arena to the global list. */
671 a->next = main_arena.next;
672 /* FIXME: The barrier is an attempt to synchronize with read access
673 in reused_arena, which does not acquire list_lock while
674 traversing the list. */
675 atomic_write_barrier ();
676 main_arena.next = a;
677
678 __libc_lock_unlock (list_lock);
679
680 __libc_lock_lock (free_list_lock);
681 detach_arena (replaced_arena);
682 __libc_lock_unlock (free_list_lock);
683
684 /* Lock this arena. NB: Another thread may have been attached to
685 this arena because the arena is now accessible from the
686 main_arena.next list and could have been picked by reused_arena.
687 This can only happen for the last arena created (before the arena
688 limit is reached). At this point, some arena has to be attached
689 to two threads. We could acquire the arena lock before list_lock
690 to make it less likely that reused_arena picks this new arena,
691 but this could result in a deadlock with
692 __malloc_fork_lock_parent. */
693
694 __libc_lock_lock (a->mutex);
695
696 return a;
697}
698
699
700/* Remove an arena from free_list. */
701static mstate
702get_free_list (void)
703{
704 mstate replaced_arena = thread_arena;
705 mstate result = free_list;
706 if (result != NULL)
707 {
708 __libc_lock_lock (free_list_lock);
709 result = free_list;
710 if (result != NULL)
711 {
712 free_list = result->next_free;
713
714 /* The arena will be attached to this thread. */
715 assert (result->attached_threads == 0);
716 result->attached_threads = 1;
717
718 detach_arena (replaced_arena);
719 }
720 __libc_lock_unlock (free_list_lock);
721
722 if (result != NULL)
723 {
724 LIBC_PROBE (memory_arena_reuse_free_list, 1, result);
725 __libc_lock_lock (result->mutex);
726 thread_arena = result;
727 }
728 }
729
730 return result;
731}
732
733/* Remove the arena from the free list (if it is present).
734 free_list_lock must have been acquired by the caller. */
735static void
736remove_from_free_list (mstate arena)
737{
738 mstate *previous = &free_list;
739 for (mstate p = free_list; p != NULL; p = p->next_free)
740 {
741 assert (p->attached_threads == 0);
742 if (p == arena)
743 {
744 /* Remove the requested arena from the list. */
745 *previous = p->next_free;
746 break;
747 }
748 else
749 previous = &p->next_free;
750 }
751}
752
753/* Lock and return an arena that can be reused for memory allocation.
754 Avoid AVOID_ARENA as we have already failed to allocate memory in
755 it and it is currently locked. */
756static mstate
757reused_arena (mstate avoid_arena)
758{
759 mstate result;
760 /* FIXME: Access to next_to_use suffers from data races. */
761 static mstate next_to_use;
762 if (next_to_use == NULL)
763 next_to_use = &main_arena;
764
765 /* Iterate over all arenas (including those linked from
766 free_list). */
767 result = next_to_use;
768 do
769 {
770 if (!__libc_lock_trylock (result->mutex))
771 goto out;
772
773 /* FIXME: This is a data race, see _int_new_arena. */
774 result = result->next;
775 }
776 while (result != next_to_use);
777
778 /* Avoid AVOID_ARENA as we have already failed to allocate memory
779 in that arena and it is currently locked. */
780 if (result == avoid_arena)
781 result = result->next;
782
783 /* No arena available without contention. Wait for the next in line. */
784 LIBC_PROBE (memory_arena_reuse_wait, 3, &result->mutex, result, avoid_arena);
785 __libc_lock_lock (result->mutex);
786
787out:
788 /* Attach the arena to the current thread. */
789 {
790 /* Update the arena thread attachment counters. */
791 mstate replaced_arena = thread_arena;
792 __libc_lock_lock (free_list_lock);
793 detach_arena (replaced_arena);
794
795 /* We may have picked up an arena on the free list. We need to
796 preserve the invariant that no arena on the free list has a
797 positive attached_threads counter (otherwise,
798 arena_thread_freeres cannot use the counter to determine if the
799 arena needs to be put on the free list). We unconditionally
800 remove the selected arena from the free list. The caller of
801 reused_arena checked the free list and observed it to be empty,
802 so the list is very short. */
803 remove_from_free_list (result);
804
805 ++result->attached_threads;
806
807 __libc_lock_unlock (free_list_lock);
808 }
809
810 LIBC_PROBE (memory_arena_reuse, 2, result, avoid_arena);
811 thread_arena = result;
812 next_to_use = result->next;
813
814 return result;
815}
816
817static mstate
818arena_get2 (size_t size, mstate avoid_arena)
819{
820 mstate a;
821
822 static size_t narenas_limit;
823
824 a = get_free_list ();
825 if (a == NULL)
826 {
827 /* Nothing immediately available, so generate a new arena. */
828 if (narenas_limit == 0)
829 {
830 if (mp_.arena_max != 0)
831 narenas_limit = mp_.arena_max;
832 else if (narenas > mp_.arena_test)
833 {
834 int n = __get_nprocs ();
835
836 if (n >= 1)
837 narenas_limit = NARENAS_FROM_NCORES (n);
838 else
839 /* We have no information about the system. Assume two
840 cores. */
841 narenas_limit = NARENAS_FROM_NCORES (2);
842 }
843 }
844 repeat:;
845 size_t n = narenas;
846 /* NB: the following depends on the fact that (size_t)0 - 1 is a
847 very large number and that the underflow is OK. If arena_max
848 is set the value of arena_test is irrelevant. If arena_test
849 is set but narenas is not yet larger or equal to arena_test
850 narenas_limit is 0. There is no possibility for narenas to
851 be too big for the test to always fail since there is not
852 enough address space to create that many arenas. */
853 if (__glibc_unlikely (n <= narenas_limit - 1))
854 {
855 if (catomic_compare_and_exchange_bool_acq (&narenas, n + 1, n))
856 goto repeat;
857 a = _int_new_arena (size);
858 if (__glibc_unlikely (a == NULL))
859 catomic_decrement (&narenas);
860 }
861 else
862 a = reused_arena (avoid_arena);
863 }
864 return a;
865}
866
867/* If we don't have the main arena, then maybe the failure is due to running
868 out of mmapped areas, so we can try allocating on the main arena.
869 Otherwise, it is likely that sbrk() has failed and there is still a chance
870 to mmap(), so try one of the other arenas. */
871static mstate
872arena_get_retry (mstate ar_ptr, size_t bytes)
873{
874 LIBC_PROBE (memory_arena_retry, 2, bytes, ar_ptr);
875 if (ar_ptr != &main_arena)
876 {
877 __libc_lock_unlock (ar_ptr->mutex);
878 ar_ptr = &main_arena;
879 __libc_lock_lock (ar_ptr->mutex);
880 }
881 else
882 {
883 __libc_lock_unlock (ar_ptr->mutex);
884 ar_ptr = arena_get2 (bytes, ar_ptr);
885 }
886
887 return ar_ptr;
888}
889#endif
890
891void
892__malloc_arena_thread_freeres (void)
893{
894 /* Shut down the thread cache first. This could deallocate data for
895 the thread arena, so do this before we put the arena on the free
896 list. */
897 tcache_thread_shutdown ();
898
899 mstate a = thread_arena;
900 thread_arena = NULL;
901
902 if (a != NULL)
903 {
904 __libc_lock_lock (free_list_lock);
905 /* If this was the last attached thread for this arena, put the
906 arena on the free list. */
907 assert (a->attached_threads > 0);
908 if (--a->attached_threads == 0)
909 {
910 a->next_free = free_list;
911 free_list = a;
912 }
913 __libc_lock_unlock (free_list_lock);
914 }
915}
916
917/*
918 * Local variables:
919 * c-basic-offset: 2
920 * End:
921 */
922

source code of glibc/malloc/arena.c