1 | //! Memory allocation APIs |
2 | |
3 | #![stable (feature = "alloc_module" , since = "1.28.0" )] |
4 | |
5 | #[stable (feature = "alloc_module" , since = "1.28.0" )] |
6 | #[doc (inline)] |
7 | pub use core::alloc::*; |
8 | use core::hint; |
9 | use core::ptr::{self, NonNull}; |
10 | |
11 | unsafe extern "Rust" { |
12 | // These are the magic symbols to call the global allocator. rustc generates |
13 | // them to call the global allocator if there is a `#[global_allocator]` attribute |
14 | // (the code expanding that attribute macro generates those functions), or to call |
15 | // the default implementations in std (`__rdl_alloc` etc. in `library/std/src/alloc.rs`) |
16 | // otherwise. |
17 | #[rustc_allocator ] |
18 | #[rustc_nounwind ] |
19 | #[cfg_attr (not(bootstrap), rustc_std_internal_symbol)] |
20 | unsafefn __rust_alloc(size: usize, align: usize) -> *mut u8; |
21 | #[rustc_deallocator ] |
22 | #[rustc_nounwind ] |
23 | #[cfg_attr (not(bootstrap), rustc_std_internal_symbol)] |
24 | unsafefn __rust_dealloc(ptr: *mut u8, size: usize, align: usize); |
25 | #[rustc_reallocator ] |
26 | #[rustc_nounwind ] |
27 | #[cfg_attr (not(bootstrap), rustc_std_internal_symbol)] |
28 | unsafefn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8; |
29 | #[rustc_allocator_zeroed ] |
30 | #[rustc_nounwind ] |
31 | #[cfg_attr (not(bootstrap), rustc_std_internal_symbol)] |
32 | unsafefn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8; |
33 | |
34 | #[cfg_attr (not(bootstrap), rustc_std_internal_symbol)] |
35 | unsafestatic __rust_no_alloc_shim_is_unstable: u8; |
36 | } |
37 | |
38 | /// The global memory allocator. |
39 | /// |
40 | /// This type implements the [`Allocator`] trait by forwarding calls |
41 | /// to the allocator registered with the `#[global_allocator]` attribute |
42 | /// if there is one, or the `std` crate’s default. |
43 | /// |
44 | /// Note: while this type is unstable, the functionality it provides can be |
45 | /// accessed through the [free functions in `alloc`](self#functions). |
46 | #[unstable (feature = "allocator_api" , issue = "32838" )] |
47 | #[derive (Copy, Clone, Default, Debug)] |
48 | // the compiler needs to know when a Box uses the global allocator vs a custom one |
49 | #[lang = "global_alloc_ty" ] |
50 | pub struct Global; |
51 | |
52 | /// Allocates memory with the global allocator. |
53 | /// |
54 | /// This function forwards calls to the [`GlobalAlloc::alloc`] method |
55 | /// of the allocator registered with the `#[global_allocator]` attribute |
56 | /// if there is one, or the `std` crate’s default. |
57 | /// |
58 | /// This function is expected to be deprecated in favor of the `allocate` method |
59 | /// of the [`Global`] type when it and the [`Allocator`] trait become stable. |
60 | /// |
61 | /// # Safety |
62 | /// |
63 | /// See [`GlobalAlloc::alloc`]. |
64 | /// |
65 | /// # Examples |
66 | /// |
67 | /// ``` |
68 | /// use std::alloc::{alloc, dealloc, handle_alloc_error, Layout}; |
69 | /// |
70 | /// unsafe { |
71 | /// let layout = Layout::new::<u16>(); |
72 | /// let ptr = alloc(layout); |
73 | /// if ptr.is_null() { |
74 | /// handle_alloc_error(layout); |
75 | /// } |
76 | /// |
77 | /// *(ptr as *mut u16) = 42; |
78 | /// assert_eq!(*(ptr as *mut u16), 42); |
79 | /// |
80 | /// dealloc(ptr, layout); |
81 | /// } |
82 | /// ``` |
83 | #[stable (feature = "global_alloc" , since = "1.28.0" )] |
84 | #[must_use = "losing the pointer will leak memory" ] |
85 | #[inline ] |
86 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
87 | pub unsafe fn alloc(layout: Layout) -> *mut u8 { |
88 | unsafe { |
89 | // Make sure we don't accidentally allow omitting the allocator shim in |
90 | // stable code until it is actually stabilized. |
91 | core::ptr::read_volatile(&__rust_no_alloc_shim_is_unstable); |
92 | |
93 | __rust_alloc(layout.size(), layout.align()) |
94 | } |
95 | } |
96 | |
97 | /// Deallocates memory with the global allocator. |
98 | /// |
99 | /// This function forwards calls to the [`GlobalAlloc::dealloc`] method |
100 | /// of the allocator registered with the `#[global_allocator]` attribute |
101 | /// if there is one, or the `std` crate’s default. |
102 | /// |
103 | /// This function is expected to be deprecated in favor of the `deallocate` method |
104 | /// of the [`Global`] type when it and the [`Allocator`] trait become stable. |
105 | /// |
106 | /// # Safety |
107 | /// |
108 | /// See [`GlobalAlloc::dealloc`]. |
109 | #[stable (feature = "global_alloc" , since = "1.28.0" )] |
110 | #[inline ] |
111 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
112 | pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) { |
113 | unsafe { __rust_dealloc(ptr, layout.size(), layout.align()) } |
114 | } |
115 | |
116 | /// Reallocates memory with the global allocator. |
117 | /// |
118 | /// This function forwards calls to the [`GlobalAlloc::realloc`] method |
119 | /// of the allocator registered with the `#[global_allocator]` attribute |
120 | /// if there is one, or the `std` crate’s default. |
121 | /// |
122 | /// This function is expected to be deprecated in favor of the `grow` and `shrink` methods |
123 | /// of the [`Global`] type when it and the [`Allocator`] trait become stable. |
124 | /// |
125 | /// # Safety |
126 | /// |
127 | /// See [`GlobalAlloc::realloc`]. |
128 | #[stable (feature = "global_alloc" , since = "1.28.0" )] |
129 | #[must_use = "losing the pointer will leak memory" ] |
130 | #[inline ] |
131 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
132 | pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { |
133 | unsafe { __rust_realloc(ptr, old_size:layout.size(), layout.align(), new_size) } |
134 | } |
135 | |
136 | /// Allocates zero-initialized memory with the global allocator. |
137 | /// |
138 | /// This function forwards calls to the [`GlobalAlloc::alloc_zeroed`] method |
139 | /// of the allocator registered with the `#[global_allocator]` attribute |
140 | /// if there is one, or the `std` crate’s default. |
141 | /// |
142 | /// This function is expected to be deprecated in favor of the `allocate_zeroed` method |
143 | /// of the [`Global`] type when it and the [`Allocator`] trait become stable. |
144 | /// |
145 | /// # Safety |
146 | /// |
147 | /// See [`GlobalAlloc::alloc_zeroed`]. |
148 | /// |
149 | /// # Examples |
150 | /// |
151 | /// ``` |
152 | /// use std::alloc::{alloc_zeroed, dealloc, handle_alloc_error, Layout}; |
153 | /// |
154 | /// unsafe { |
155 | /// let layout = Layout::new::<u16>(); |
156 | /// let ptr = alloc_zeroed(layout); |
157 | /// if ptr.is_null() { |
158 | /// handle_alloc_error(layout); |
159 | /// } |
160 | /// |
161 | /// assert_eq!(*(ptr as *mut u16), 0); |
162 | /// |
163 | /// dealloc(ptr, layout); |
164 | /// } |
165 | /// ``` |
166 | #[stable (feature = "global_alloc" , since = "1.28.0" )] |
167 | #[must_use = "losing the pointer will leak memory" ] |
168 | #[inline ] |
169 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
170 | pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 { |
171 | unsafe { |
172 | // Make sure we don't accidentally allow omitting the allocator shim in |
173 | // stable code until it is actually stabilized. |
174 | core::ptr::read_volatile(&__rust_no_alloc_shim_is_unstable); |
175 | |
176 | __rust_alloc_zeroed(layout.size(), layout.align()) |
177 | } |
178 | } |
179 | |
180 | impl Global { |
181 | #[inline ] |
182 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
183 | fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result<NonNull<[u8]>, AllocError> { |
184 | match layout.size() { |
185 | 0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)), |
186 | // SAFETY: `layout` is non-zero in size, |
187 | size => unsafe { |
188 | let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) }; |
189 | let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; |
190 | Ok(NonNull::slice_from_raw_parts(ptr, size)) |
191 | }, |
192 | } |
193 | } |
194 | |
195 | // SAFETY: Same as `Allocator::grow` |
196 | #[inline ] |
197 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
198 | unsafe fn grow_impl( |
199 | &self, |
200 | ptr: NonNull<u8>, |
201 | old_layout: Layout, |
202 | new_layout: Layout, |
203 | zeroed: bool, |
204 | ) -> Result<NonNull<[u8]>, AllocError> { |
205 | debug_assert!( |
206 | new_layout.size() >= old_layout.size(), |
207 | "`new_layout.size()` must be greater than or equal to `old_layout.size()`" |
208 | ); |
209 | |
210 | match old_layout.size() { |
211 | 0 => self.alloc_impl(new_layout, zeroed), |
212 | |
213 | // SAFETY: `new_size` is non-zero as `old_size` is greater than or equal to `new_size` |
214 | // as required by safety conditions. Other conditions must be upheld by the caller |
215 | old_size if old_layout.align() == new_layout.align() => unsafe { |
216 | let new_size = new_layout.size(); |
217 | |
218 | // `realloc` probably checks for `new_size >= old_layout.size()` or something similar. |
219 | hint::assert_unchecked(new_size >= old_layout.size()); |
220 | |
221 | let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); |
222 | let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; |
223 | if zeroed { |
224 | raw_ptr.add(old_size).write_bytes(0, new_size - old_size); |
225 | } |
226 | Ok(NonNull::slice_from_raw_parts(ptr, new_size)) |
227 | }, |
228 | |
229 | // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`, |
230 | // both the old and new memory allocation are valid for reads and writes for `old_size` |
231 | // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap |
232 | // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract |
233 | // for `dealloc` must be upheld by the caller. |
234 | old_size => unsafe { |
235 | let new_ptr = self.alloc_impl(new_layout, zeroed)?; |
236 | ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size); |
237 | self.deallocate(ptr, old_layout); |
238 | Ok(new_ptr) |
239 | }, |
240 | } |
241 | } |
242 | } |
243 | |
244 | #[unstable (feature = "allocator_api" , issue = "32838" )] |
245 | unsafe impl Allocator for Global { |
246 | #[inline ] |
247 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
248 | fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> { |
249 | self.alloc_impl(layout, false) |
250 | } |
251 | |
252 | #[inline ] |
253 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
254 | fn allocate_zeroed(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> { |
255 | self.alloc_impl(layout, true) |
256 | } |
257 | |
258 | #[inline ] |
259 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
260 | unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) { |
261 | if layout.size() != 0 { |
262 | // SAFETY: |
263 | // * We have checked that `layout` is non-zero in size. |
264 | // * The caller is obligated to provide a layout that "fits", and in this case, |
265 | // "fit" always means a layout that is equal to the original, because our |
266 | // `allocate()`, `grow()`, and `shrink()` implementations never returns a larger |
267 | // allocation than requested. |
268 | // * Other conditions must be upheld by the caller, as per `Allocator::deallocate()`'s |
269 | // safety documentation. |
270 | unsafe { dealloc(ptr.as_ptr(), layout) } |
271 | } |
272 | } |
273 | |
274 | #[inline ] |
275 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
276 | unsafe fn grow( |
277 | &self, |
278 | ptr: NonNull<u8>, |
279 | old_layout: Layout, |
280 | new_layout: Layout, |
281 | ) -> Result<NonNull<[u8]>, AllocError> { |
282 | // SAFETY: all conditions must be upheld by the caller |
283 | unsafe { self.grow_impl(ptr, old_layout, new_layout, false) } |
284 | } |
285 | |
286 | #[inline ] |
287 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
288 | unsafe fn grow_zeroed( |
289 | &self, |
290 | ptr: NonNull<u8>, |
291 | old_layout: Layout, |
292 | new_layout: Layout, |
293 | ) -> Result<NonNull<[u8]>, AllocError> { |
294 | // SAFETY: all conditions must be upheld by the caller |
295 | unsafe { self.grow_impl(ptr, old_layout, new_layout, true) } |
296 | } |
297 | |
298 | #[inline ] |
299 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
300 | unsafe fn shrink( |
301 | &self, |
302 | ptr: NonNull<u8>, |
303 | old_layout: Layout, |
304 | new_layout: Layout, |
305 | ) -> Result<NonNull<[u8]>, AllocError> { |
306 | debug_assert!( |
307 | new_layout.size() <= old_layout.size(), |
308 | "`new_layout.size()` must be smaller than or equal to `old_layout.size()`" |
309 | ); |
310 | |
311 | match new_layout.size() { |
312 | // SAFETY: conditions must be upheld by the caller |
313 | 0 => unsafe { |
314 | self.deallocate(ptr, old_layout); |
315 | Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0)) |
316 | }, |
317 | |
318 | // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller |
319 | new_size if old_layout.align() == new_layout.align() => unsafe { |
320 | // `realloc` probably checks for `new_size <= old_layout.size()` or something similar. |
321 | hint::assert_unchecked(new_size <= old_layout.size()); |
322 | |
323 | let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size); |
324 | let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?; |
325 | Ok(NonNull::slice_from_raw_parts(ptr, new_size)) |
326 | }, |
327 | |
328 | // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`, |
329 | // both the old and new memory allocation are valid for reads and writes for `new_size` |
330 | // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap |
331 | // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract |
332 | // for `dealloc` must be upheld by the caller. |
333 | new_size => unsafe { |
334 | let new_ptr = self.allocate(new_layout)?; |
335 | ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size); |
336 | self.deallocate(ptr, old_layout); |
337 | Ok(new_ptr) |
338 | }, |
339 | } |
340 | } |
341 | } |
342 | |
343 | /// The allocator for `Box`. |
344 | #[cfg (not(no_global_oom_handling))] |
345 | #[lang = "exchange_malloc" ] |
346 | #[inline ] |
347 | #[cfg_attr (miri, track_caller)] // even without panics, this helps for Miri backtraces |
348 | unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 { |
349 | let layout: Layout = unsafe { Layout::from_size_align_unchecked(size, align) }; |
350 | match Global.allocate(layout) { |
351 | Ok(ptr: NonNull<[u8]>) => ptr.as_mut_ptr(), |
352 | Err(_) => handle_alloc_error(layout), |
353 | } |
354 | } |
355 | |
356 | // # Allocation error handler |
357 | |
358 | #[cfg (not(no_global_oom_handling))] |
359 | unsafe extern "Rust" { |
360 | // This is the magic symbol to call the global alloc error handler. rustc generates |
361 | // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the |
362 | // default implementations below (`__rdl_oom`) otherwise. |
363 | #[cfg_attr (not(bootstrap), rustc_std_internal_symbol)] |
364 | unsafefn __rust_alloc_error_handler(size: usize, align: usize) -> !; |
365 | } |
366 | |
367 | /// Signals a memory allocation error. |
368 | /// |
369 | /// Callers of memory allocation APIs wishing to cease execution |
370 | /// in response to an allocation error are encouraged to call this function, |
371 | /// rather than directly invoking [`panic!`] or similar. |
372 | /// |
373 | /// This function is guaranteed to diverge (not return normally with a value), but depending on |
374 | /// global configuration, it may either panic (resulting in unwinding or aborting as per |
375 | /// configuration for all panics), or abort the process (with no unwinding). |
376 | /// |
377 | /// The default behavior is: |
378 | /// |
379 | /// * If the binary links against `std` (typically the case), then |
380 | /// print a message to standard error and abort the process. |
381 | /// This behavior can be replaced with [`set_alloc_error_hook`] and [`take_alloc_error_hook`]. |
382 | /// Future versions of Rust may panic by default instead. |
383 | /// |
384 | /// * If the binary does not link against `std` (all of its crates are marked |
385 | /// [`#![no_std]`][no_std]), then call [`panic!`] with a message. |
386 | /// [The panic handler] applies as to any panic. |
387 | /// |
388 | /// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html |
389 | /// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html |
390 | /// [The panic handler]: https://doc.rust-lang.org/reference/runtime.html#the-panic_handler-attribute |
391 | /// [no_std]: https://doc.rust-lang.org/reference/names/preludes.html#the-no_std-attribute |
392 | #[stable (feature = "global_alloc" , since = "1.28.0" )] |
393 | #[rustc_const_unstable (feature = "const_alloc_error" , issue = "92523" )] |
394 | #[cfg (not(no_global_oom_handling))] |
395 | #[cold ] |
396 | #[optimize (size)] |
397 | pub const fn handle_alloc_error(layout: Layout) -> ! { |
398 | const fn ct_error(_: Layout) -> ! { |
399 | panic!("allocation failed" ); |
400 | } |
401 | |
402 | #[inline ] |
403 | fn rt_error(layout: Layout) -> ! { |
404 | unsafe { |
405 | __rust_alloc_error_handler(layout.size(), layout.align()); |
406 | } |
407 | } |
408 | |
409 | #[cfg (not(feature = "panic_immediate_abort" ))] |
410 | { |
411 | core::intrinsics::const_eval_select((layout,), _called_in_const:ct_error, _called_at_rt:rt_error) |
412 | } |
413 | |
414 | #[cfg (feature = "panic_immediate_abort" )] |
415 | ct_error(layout) |
416 | } |
417 | |
418 | #[cfg (not(no_global_oom_handling))] |
419 | #[doc (hidden)] |
420 | #[allow (unused_attributes)] |
421 | #[unstable (feature = "alloc_internals" , issue = "none" )] |
422 | pub mod __alloc_error_handler { |
423 | // called via generated `__rust_alloc_error_handler` if there is no |
424 | // `#[alloc_error_handler]`. |
425 | #[rustc_std_internal_symbol ] |
426 | pub unsafe fn __rdl_oom(size: usize, _align: usize) -> ! { |
427 | unsafe extern "Rust" { |
428 | // This symbol is emitted by rustc next to __rust_alloc_error_handler. |
429 | // Its value depends on the -Zoom={panic,abort} compiler option. |
430 | #[cfg_attr (not(bootstrap), rustc_std_internal_symbol)] |
431 | unsafestatic __rust_alloc_error_handler_should_panic: u8; |
432 | } |
433 | |
434 | if unsafe { __rust_alloc_error_handler_should_panic != 0 } { |
435 | panic!("memory allocation of {size} bytes failed" ) |
436 | } else { |
437 | core::panicking::panic_nounwind_fmt( |
438 | fmt:format_args!("memory allocation of {size} bytes failed" ), |
439 | /* force_no_backtrace */ force_no_backtrace:false, |
440 | ) |
441 | } |
442 | } |
443 | } |
444 | |