1//! Memory allocation APIs
2
3#![stable(feature = "alloc_module", since = "1.28.0")]
4
5#[stable(feature = "alloc_module", since = "1.28.0")]
6#[doc(inline)]
7pub use core::alloc::*;
8use core::hint;
9use core::ptr::{self, NonNull};
10
11unsafe extern "Rust" {
12 // These are the magic symbols to call the global allocator. rustc generates
13 // them to call the global allocator if there is a `#[global_allocator]` attribute
14 // (the code expanding that attribute macro generates those functions), or to call
15 // the default implementations in std (`__rdl_alloc` etc. in `library/std/src/alloc.rs`)
16 // otherwise.
17 #[rustc_allocator]
18 #[rustc_nounwind]
19 #[cfg_attr(not(bootstrap), rustc_std_internal_symbol)]
20 unsafefn __rust_alloc(size: usize, align: usize) -> *mut u8;
21 #[rustc_deallocator]
22 #[rustc_nounwind]
23 #[cfg_attr(not(bootstrap), rustc_std_internal_symbol)]
24 unsafefn __rust_dealloc(ptr: *mut u8, size: usize, align: usize);
25 #[rustc_reallocator]
26 #[rustc_nounwind]
27 #[cfg_attr(not(bootstrap), rustc_std_internal_symbol)]
28 unsafefn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8;
29 #[rustc_allocator_zeroed]
30 #[rustc_nounwind]
31 #[cfg_attr(not(bootstrap), rustc_std_internal_symbol)]
32 unsafefn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8;
33
34 #[cfg_attr(not(bootstrap), rustc_std_internal_symbol)]
35 unsafestatic __rust_no_alloc_shim_is_unstable: u8;
36}
37
38/// The global memory allocator.
39///
40/// This type implements the [`Allocator`] trait by forwarding calls
41/// to the allocator registered with the `#[global_allocator]` attribute
42/// if there is one, or the `std` crate’s default.
43///
44/// Note: while this type is unstable, the functionality it provides can be
45/// accessed through the [free functions in `alloc`](self#functions).
46#[unstable(feature = "allocator_api", issue = "32838")]
47#[derive(Copy, Clone, Default, Debug)]
48// the compiler needs to know when a Box uses the global allocator vs a custom one
49#[lang = "global_alloc_ty"]
50pub struct Global;
51
52/// Allocates memory with the global allocator.
53///
54/// This function forwards calls to the [`GlobalAlloc::alloc`] method
55/// of the allocator registered with the `#[global_allocator]` attribute
56/// if there is one, or the `std` crate’s default.
57///
58/// This function is expected to be deprecated in favor of the `allocate` method
59/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
60///
61/// # Safety
62///
63/// See [`GlobalAlloc::alloc`].
64///
65/// # Examples
66///
67/// ```
68/// use std::alloc::{alloc, dealloc, handle_alloc_error, Layout};
69///
70/// unsafe {
71/// let layout = Layout::new::<u16>();
72/// let ptr = alloc(layout);
73/// if ptr.is_null() {
74/// handle_alloc_error(layout);
75/// }
76///
77/// *(ptr as *mut u16) = 42;
78/// assert_eq!(*(ptr as *mut u16), 42);
79///
80/// dealloc(ptr, layout);
81/// }
82/// ```
83#[stable(feature = "global_alloc", since = "1.28.0")]
84#[must_use = "losing the pointer will leak memory"]
85#[inline]
86#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
87pub unsafe fn alloc(layout: Layout) -> *mut u8 {
88 unsafe {
89 // Make sure we don't accidentally allow omitting the allocator shim in
90 // stable code until it is actually stabilized.
91 core::ptr::read_volatile(&__rust_no_alloc_shim_is_unstable);
92
93 __rust_alloc(layout.size(), layout.align())
94 }
95}
96
97/// Deallocates memory with the global allocator.
98///
99/// This function forwards calls to the [`GlobalAlloc::dealloc`] method
100/// of the allocator registered with the `#[global_allocator]` attribute
101/// if there is one, or the `std` crate’s default.
102///
103/// This function is expected to be deprecated in favor of the `deallocate` method
104/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
105///
106/// # Safety
107///
108/// See [`GlobalAlloc::dealloc`].
109#[stable(feature = "global_alloc", since = "1.28.0")]
110#[inline]
111#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
112pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) {
113 unsafe { __rust_dealloc(ptr, layout.size(), layout.align()) }
114}
115
116/// Reallocates memory with the global allocator.
117///
118/// This function forwards calls to the [`GlobalAlloc::realloc`] method
119/// of the allocator registered with the `#[global_allocator]` attribute
120/// if there is one, or the `std` crate’s default.
121///
122/// This function is expected to be deprecated in favor of the `grow` and `shrink` methods
123/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
124///
125/// # Safety
126///
127/// See [`GlobalAlloc::realloc`].
128#[stable(feature = "global_alloc", since = "1.28.0")]
129#[must_use = "losing the pointer will leak memory"]
130#[inline]
131#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
132pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
133 unsafe { __rust_realloc(ptr, old_size:layout.size(), layout.align(), new_size) }
134}
135
136/// Allocates zero-initialized memory with the global allocator.
137///
138/// This function forwards calls to the [`GlobalAlloc::alloc_zeroed`] method
139/// of the allocator registered with the `#[global_allocator]` attribute
140/// if there is one, or the `std` crate’s default.
141///
142/// This function is expected to be deprecated in favor of the `allocate_zeroed` method
143/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
144///
145/// # Safety
146///
147/// See [`GlobalAlloc::alloc_zeroed`].
148///
149/// # Examples
150///
151/// ```
152/// use std::alloc::{alloc_zeroed, dealloc, handle_alloc_error, Layout};
153///
154/// unsafe {
155/// let layout = Layout::new::<u16>();
156/// let ptr = alloc_zeroed(layout);
157/// if ptr.is_null() {
158/// handle_alloc_error(layout);
159/// }
160///
161/// assert_eq!(*(ptr as *mut u16), 0);
162///
163/// dealloc(ptr, layout);
164/// }
165/// ```
166#[stable(feature = "global_alloc", since = "1.28.0")]
167#[must_use = "losing the pointer will leak memory"]
168#[inline]
169#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
170pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 {
171 unsafe {
172 // Make sure we don't accidentally allow omitting the allocator shim in
173 // stable code until it is actually stabilized.
174 core::ptr::read_volatile(&__rust_no_alloc_shim_is_unstable);
175
176 __rust_alloc_zeroed(layout.size(), layout.align())
177 }
178}
179
180impl Global {
181 #[inline]
182 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
183 fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result<NonNull<[u8]>, AllocError> {
184 match layout.size() {
185 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)),
186 // SAFETY: `layout` is non-zero in size,
187 size => unsafe {
188 let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) };
189 let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
190 Ok(NonNull::slice_from_raw_parts(ptr, size))
191 },
192 }
193 }
194
195 // SAFETY: Same as `Allocator::grow`
196 #[inline]
197 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
198 unsafe fn grow_impl(
199 &self,
200 ptr: NonNull<u8>,
201 old_layout: Layout,
202 new_layout: Layout,
203 zeroed: bool,
204 ) -> Result<NonNull<[u8]>, AllocError> {
205 debug_assert!(
206 new_layout.size() >= old_layout.size(),
207 "`new_layout.size()` must be greater than or equal to `old_layout.size()`"
208 );
209
210 match old_layout.size() {
211 0 => self.alloc_impl(new_layout, zeroed),
212
213 // SAFETY: `new_size` is non-zero as `old_size` is greater than or equal to `new_size`
214 // as required by safety conditions. Other conditions must be upheld by the caller
215 old_size if old_layout.align() == new_layout.align() => unsafe {
216 let new_size = new_layout.size();
217
218 // `realloc` probably checks for `new_size >= old_layout.size()` or something similar.
219 hint::assert_unchecked(new_size >= old_layout.size());
220
221 let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size);
222 let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
223 if zeroed {
224 raw_ptr.add(old_size).write_bytes(0, new_size - old_size);
225 }
226 Ok(NonNull::slice_from_raw_parts(ptr, new_size))
227 },
228
229 // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`,
230 // both the old and new memory allocation are valid for reads and writes for `old_size`
231 // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap
232 // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract
233 // for `dealloc` must be upheld by the caller.
234 old_size => unsafe {
235 let new_ptr = self.alloc_impl(new_layout, zeroed)?;
236 ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size);
237 self.deallocate(ptr, old_layout);
238 Ok(new_ptr)
239 },
240 }
241 }
242}
243
244#[unstable(feature = "allocator_api", issue = "32838")]
245unsafe impl Allocator for Global {
246 #[inline]
247 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
248 fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
249 self.alloc_impl(layout, false)
250 }
251
252 #[inline]
253 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
254 fn allocate_zeroed(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
255 self.alloc_impl(layout, true)
256 }
257
258 #[inline]
259 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
260 unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
261 if layout.size() != 0 {
262 // SAFETY:
263 // * We have checked that `layout` is non-zero in size.
264 // * The caller is obligated to provide a layout that "fits", and in this case,
265 // "fit" always means a layout that is equal to the original, because our
266 // `allocate()`, `grow()`, and `shrink()` implementations never returns a larger
267 // allocation than requested.
268 // * Other conditions must be upheld by the caller, as per `Allocator::deallocate()`'s
269 // safety documentation.
270 unsafe { dealloc(ptr.as_ptr(), layout) }
271 }
272 }
273
274 #[inline]
275 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
276 unsafe fn grow(
277 &self,
278 ptr: NonNull<u8>,
279 old_layout: Layout,
280 new_layout: Layout,
281 ) -> Result<NonNull<[u8]>, AllocError> {
282 // SAFETY: all conditions must be upheld by the caller
283 unsafe { self.grow_impl(ptr, old_layout, new_layout, false) }
284 }
285
286 #[inline]
287 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
288 unsafe fn grow_zeroed(
289 &self,
290 ptr: NonNull<u8>,
291 old_layout: Layout,
292 new_layout: Layout,
293 ) -> Result<NonNull<[u8]>, AllocError> {
294 // SAFETY: all conditions must be upheld by the caller
295 unsafe { self.grow_impl(ptr, old_layout, new_layout, true) }
296 }
297
298 #[inline]
299 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
300 unsafe fn shrink(
301 &self,
302 ptr: NonNull<u8>,
303 old_layout: Layout,
304 new_layout: Layout,
305 ) -> Result<NonNull<[u8]>, AllocError> {
306 debug_assert!(
307 new_layout.size() <= old_layout.size(),
308 "`new_layout.size()` must be smaller than or equal to `old_layout.size()`"
309 );
310
311 match new_layout.size() {
312 // SAFETY: conditions must be upheld by the caller
313 0 => unsafe {
314 self.deallocate(ptr, old_layout);
315 Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0))
316 },
317
318 // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller
319 new_size if old_layout.align() == new_layout.align() => unsafe {
320 // `realloc` probably checks for `new_size <= old_layout.size()` or something similar.
321 hint::assert_unchecked(new_size <= old_layout.size());
322
323 let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size);
324 let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
325 Ok(NonNull::slice_from_raw_parts(ptr, new_size))
326 },
327
328 // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`,
329 // both the old and new memory allocation are valid for reads and writes for `new_size`
330 // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap
331 // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract
332 // for `dealloc` must be upheld by the caller.
333 new_size => unsafe {
334 let new_ptr = self.allocate(new_layout)?;
335 ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size);
336 self.deallocate(ptr, old_layout);
337 Ok(new_ptr)
338 },
339 }
340 }
341}
342
343/// The allocator for `Box`.
344#[cfg(not(no_global_oom_handling))]
345#[lang = "exchange_malloc"]
346#[inline]
347#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
348unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 {
349 let layout: Layout = unsafe { Layout::from_size_align_unchecked(size, align) };
350 match Global.allocate(layout) {
351 Ok(ptr: NonNull<[u8]>) => ptr.as_mut_ptr(),
352 Err(_) => handle_alloc_error(layout),
353 }
354}
355
356// # Allocation error handler
357
358#[cfg(not(no_global_oom_handling))]
359unsafe extern "Rust" {
360 // This is the magic symbol to call the global alloc error handler. rustc generates
361 // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the
362 // default implementations below (`__rdl_oom`) otherwise.
363 #[cfg_attr(not(bootstrap), rustc_std_internal_symbol)]
364 unsafefn __rust_alloc_error_handler(size: usize, align: usize) -> !;
365}
366
367/// Signals a memory allocation error.
368///
369/// Callers of memory allocation APIs wishing to cease execution
370/// in response to an allocation error are encouraged to call this function,
371/// rather than directly invoking [`panic!`] or similar.
372///
373/// This function is guaranteed to diverge (not return normally with a value), but depending on
374/// global configuration, it may either panic (resulting in unwinding or aborting as per
375/// configuration for all panics), or abort the process (with no unwinding).
376///
377/// The default behavior is:
378///
379/// * If the binary links against `std` (typically the case), then
380/// print a message to standard error and abort the process.
381/// This behavior can be replaced with [`set_alloc_error_hook`] and [`take_alloc_error_hook`].
382/// Future versions of Rust may panic by default instead.
383///
384/// * If the binary does not link against `std` (all of its crates are marked
385/// [`#![no_std]`][no_std]), then call [`panic!`] with a message.
386/// [The panic handler] applies as to any panic.
387///
388/// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html
389/// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html
390/// [The panic handler]: https://doc.rust-lang.org/reference/runtime.html#the-panic_handler-attribute
391/// [no_std]: https://doc.rust-lang.org/reference/names/preludes.html#the-no_std-attribute
392#[stable(feature = "global_alloc", since = "1.28.0")]
393#[rustc_const_unstable(feature = "const_alloc_error", issue = "92523")]
394#[cfg(not(no_global_oom_handling))]
395#[cold]
396#[optimize(size)]
397pub const fn handle_alloc_error(layout: Layout) -> ! {
398 const fn ct_error(_: Layout) -> ! {
399 panic!("allocation failed");
400 }
401
402 #[inline]
403 fn rt_error(layout: Layout) -> ! {
404 unsafe {
405 __rust_alloc_error_handler(layout.size(), layout.align());
406 }
407 }
408
409 #[cfg(not(feature = "panic_immediate_abort"))]
410 {
411 core::intrinsics::const_eval_select((layout,), _called_in_const:ct_error, _called_at_rt:rt_error)
412 }
413
414 #[cfg(feature = "panic_immediate_abort")]
415 ct_error(layout)
416}
417
418#[cfg(not(no_global_oom_handling))]
419#[doc(hidden)]
420#[allow(unused_attributes)]
421#[unstable(feature = "alloc_internals", issue = "none")]
422pub mod __alloc_error_handler {
423 // called via generated `__rust_alloc_error_handler` if there is no
424 // `#[alloc_error_handler]`.
425 #[rustc_std_internal_symbol]
426 pub unsafe fn __rdl_oom(size: usize, _align: usize) -> ! {
427 unsafe extern "Rust" {
428 // This symbol is emitted by rustc next to __rust_alloc_error_handler.
429 // Its value depends on the -Zoom={panic,abort} compiler option.
430 #[cfg_attr(not(bootstrap), rustc_std_internal_symbol)]
431 unsafestatic __rust_alloc_error_handler_should_panic: u8;
432 }
433
434 if unsafe { __rust_alloc_error_handler_should_panic != 0 } {
435 panic!("memory allocation of {size} bytes failed")
436 } else {
437 core::panicking::panic_nounwind_fmt(
438 fmt:format_args!("memory allocation of {size} bytes failed"),
439 /* force_no_backtrace */ force_no_backtrace:false,
440 )
441 }
442 }
443}
444