ptr.rs source code [crates/zerocopy-0.8.14/src/pointer/ptr.rs]

1	// Copyright 2023 The Fuchsia Authors
2	//
3	// Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
4	// <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
5	// license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
6	// This file may not be copied, modified, or distributed except according to
7	// those terms.
8
9	use core::ptr::NonNull;
10
11	use crate::{util::AsAddress, CastType, KnownLayout};
12
13	/// Module used to gate access to [`Ptr`]'s fields.
14	mod def {
15	#[cfg(doc)]
16	use super::invariant;
17	use super::Invariants;
18	use core::{marker::PhantomData, ptr::NonNull};
19
20	/// A raw pointer with more restrictions.
21	///
22	/// `Ptr<T>` is similar to [`NonNull<T>`], but it is more restrictive in the
23	/// following ways (note that these requirements only hold of non-zero-sized
24	/// referents):
25	/// - It must derive from a valid allocation.
26	/// - It must reference a byte range which is contained inside the
27	/// allocation from which it derives.
28	/// - As a consequence, the byte range it references must have a size
29	/// which does not overflow `isize`.
30	///
31	/// Depending on how `Ptr` is parameterized, it may have additional
32	/// invariants:
33	/// - `ptr` conforms to the aliasing invariant of
34	/// [`I::Aliasing`](invariant::Aliasing).
35	/// - `ptr` conforms to the alignment invariant of
36	/// [`I::Alignment`](invariant::Alignment).
37	/// - `ptr` conforms to the validity invariant of
38	/// [`I::Validity`](invariant::Validity).
39	///
40	/// `Ptr<'a, T>` is [covariant] in `'a` and `T`.
41	///
42	/// [covariant]: https://doc.rust-lang.org/reference/subtyping.html
43	pub struct Ptr<'a, T, I>
44	where
45	T: 'a + ?Sized,
46	I: Invariants,
47	{
48	/// # Invariants
49	///
50	/// 0. If `ptr`'s referent is not zero sized, then `ptr` is derived from
51	/// some valid Rust allocation, `A`.
52	/// 1. If `ptr`'s referent is not zero sized, then `ptr` has valid
53	/// provenance for `A`.
54	/// 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a
55	/// byte range which is entirely contained in `A`.
56	/// 3. `ptr` addresses a byte range whose length fits in an `isize`.
57	/// 4. `ptr` addresses a byte range which does not wrap around the
58	/// address space.
59	/// 5. If `ptr`'s referent is not zero sized,`A` is guaranteed to live
60	/// for at least `'a`.
61	/// 6. `T: 'a`.
62	/// 7. `ptr` conforms to the aliasing invariant of
63	/// [`I::Aliasing`](invariant::Aliasing).
64	/// 8. `ptr` conforms to the alignment invariant of
65	/// [`I::Alignment`](invariant::Alignment).
66	/// 9. `ptr` conforms to the validity invariant of
67	/// [`I::Validity`](invariant::Validity).
68	// SAFETY: `NonNull<T>` is covariant over `T` [1].
69	//
70	// [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html
71	ptr: NonNull<T>,
72	// SAFETY: `&'a ()` is covariant over `'a` [1].
73	//
74	// [1]: https://doc.rust-lang.org/reference/subtyping.html#variance
75	_invariants: PhantomData<&'a I>,
76	}
77
78	impl<'a, T, I> Ptr<'a, T, I>
79	where
80	T: 'a + ?Sized,
81	I: Invariants,
82	{
83	/// Constructs a `Ptr` from a [`NonNull`].
84	///
85	/// # Safety
86	///
87	/// The caller promises that:
88	///
89	/// 0. If `ptr`'s referent is not zero sized, then `ptr` is derived from
90	/// some valid Rust allocation, `A`.
91	/// 1. If `ptr`'s referent is not zero sized, then `ptr` has valid
92	/// provenance for `A`.
93	/// 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a
94	/// byte range which is entirely contained in `A`.
95	/// 3. `ptr` addresses a byte range whose length fits in an `isize`.
96	/// 4. `ptr` addresses a byte range which does not wrap around the
97	/// address space.
98	/// 5. If `ptr`'s referent is not zero sized, then `A` is guaranteed to
99	/// live for at least `'a`.
100	/// 6. `ptr` conforms to the aliasing invariant of
101	/// [`I::Aliasing`](invariant::Aliasing).
102	/// 7. `ptr` conforms to the alignment invariant of
103	/// [`I::Alignment`](invariant::Alignment).
104	/// 8. `ptr` conforms to the validity invariant of
105	/// [`I::Validity`](invariant::Validity).
106	pub(super) unsafe fn new(ptr: NonNull<T>) -> Ptr<'a, T, I> {
107	// SAFETY: The caller has promised to satisfy all safety invariants
108	// of `Ptr`.
109	Self { ptr, _invariants: PhantomData }
110	}
111
112	/// Converts this `Ptr<T>` to a [`NonNull<T>`].
113	///
114	/// Note that this method does not consume `self`. The caller should
115	/// watch out for `unsafe` code which uses the returned `NonNull` in a
116	/// way that violates the safety invariants of `self`.
117	pub(crate) fn as_non_null(&self) -> NonNull<T> {
118	self.ptr
119	}
120	}
121	}
122
123	#[allow(unreachable_pub)] // This is a false positive on our MSRV toolchain.
124	pub use def::Ptr;
125
126	/// Used to define the system of [invariants][invariant] of `Ptr`.
127	macro_rules! define_system {
128	($(#[$system_attr:meta])* $system:ident {
129	$($(#[$set_attr:meta])* $set:ident {
130	$( $(#[$elem_attr:meta])* $elem:ident $(< $($stronger_elem:ident)\|)?,)
131	})*
132	}) => {
133	/// No requirement - any invariant is allowed.
134	#[allow(missing_copy_implementations, missing_debug_implementations)]
135	pub enum Any {}
136
137	/// `Self` imposes a requirement at least as strict as `I`.
138	pub trait AtLeast<I> {}
139
140	mod sealed {
141	pub trait Sealed {}
142
143	impl<$($set,)> Sealed for* ($($set,)*)
144	where
145	$($set: super::$set,)*
146	{}
147
148	impl Sealed for super::Any {}
149
150	$($(
151	impl Sealed for super::$elem {}
152	))
153	}
154
155	$(#[$system_attr])*
156	///
157	#[doc = concat!(
158	stringify!($system),
159	" are encoded as tuples of (",
160	)]
161	$(#[doc = concat!(
162	"[`",
163	stringify!($set),
164	"`],"
165	)])*
166	#[doc = concat!(
167	").",
168	)]
169	/// This trait is implemented for such tuples, and can be used to
170	/// project out the components of these tuples via its associated types.
171	pub trait $system: sealed::Sealed {
172	$(
173	$(#[$set_attr])*
174	type $set: $set;
175	)*
176	}
177
178	impl<$($set,)> $system for* ($($set,)*)
179	where
180	$($set: self::$set,)*
181	{
182	$(type $set = $set;)*
183	}
184
185	$(
186	$(#[$set_attr])*
187	pub trait $set: 'static + sealed::Sealed {
188	// This only exists for use in
189	// `into_exclusive_or_post_monomorphization_error`.
190	#[doc(hidden)]
191	const NAME: &'static str;
192	}
193
194	impl $set for Any {
195	const NAME: &'static str = stringify!(Any);
196	}
197
198	$(
199	$(#[$elem_attr])*
200	#[allow(missing_copy_implementations, missing_debug_implementations)]
201	pub enum $elem {}
202
203	$(#[$elem_attr])*
204	impl $set for $elem {
205	const NAME: &'static str = stringify!($elem);
206	}
207	)*
208	)*
209
210	$($(
211	impl AtLeast<Any> for $elem {}
212	impl AtLeast<$elem> for $elem {}
213
214	$($(impl AtLeast<$elem> for $stronger_elem {})*)?
215	))
216	};
217	}
218
219	/// The parameterized invariants of a [`Ptr`].
220	///
221	/// Invariants are encoded as ([`Aliasing`], [`Alignment`], [`Validity`])
222	/// triples implementing the [`Invariants`] trait.
223	#[doc(hidden)]
224	pub mod invariant {
225	define_system! {
226	/// The invariants of a [`Ptr`][super::Ptr].
227	Invariants {
228	/// The aliasing invariant of a [`Ptr`][super::Ptr].
229	Aliasing {
230	/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a T`.
231	///
232	/// The referent of a shared-aliased `Ptr` may be concurrently
233	/// referenced by any number of shared-aliased `Ptr` or `&T`
234	/// references, and may not be concurrently referenced by any
235	/// exclusively-aliased `Ptr`s or `&mut T` references. The
236	/// referent must not be mutated, except via [`UnsafeCell`]s.
237	///
238	/// [`UnsafeCell`]: core::cell::UnsafeCell
239	Shared < Exclusive,
240
241	/// The `Ptr<'a, T>` adheres to the aliasing rules of a `&'a mut
242	/// T`.
243	///
244	/// The referent of an exclusively-aliased `Ptr` may not be
245	/// concurrently referenced by any other `Ptr`s or references,
246	/// and may not be accessed (read or written) other than via
247	/// this `Ptr`.
248	Exclusive,
249	}
250
251	/// The alignment invariant of a [`Ptr`][super::Ptr].
252	Alignment {
253	/// The referent is aligned: for `Ptr<T>`, the referent's
254	/// address is a multiple of the `T`'s alignment.
255	Aligned,
256	}
257
258	/// The validity invariant of a [`Ptr`][super::Ptr].
259	Validity {
260	/// The byte ranges initialized in `T` are also initialized in
261	/// the referent.
262	///
263	/// Formally: uninitialized bytes may only be present in
264	/// `Ptr<T>`'s referent where they are guaranteed to be present
265	/// in `T`. This is a dynamic property: if, at a particular byte
266	/// offset, a valid enum discriminant is set, the subsequent
267	/// bytes may only have uninitialized bytes as specificed by the
268	/// corresponding enum.
269	///
270	/// Formally, given `len = size_of_val_raw(ptr)`, at every byte
271	/// offset, `b`, in the range `[0, len)`:
272	/// - If, in any instance `t: T` of length `len`, the byte at
273	/// offset `b` in `t` is initialized, then the byte at offset
274	/// `b` within `ptr` must be initialized.*
275	/// - Let `c` be the contents of the byte range `[0, b)` in
276	/// `ptr`. Let `S` be the subset of valid instances of `T` of*
277	/// length `len` which contain `c` in the offset range `[0,
278	/// b)`. If, in any instance of `t: T` in `S`, the byte at
279	/// offset `b` in `t` is initialized, then the byte at offset
280	/// `b` in `ptr` must be initialized.*
281	///
282	/// Pragmatically, this means that if `ptr` is guaranteed to*
283	/// contain an enum type at a particular offset, and the enum
284	/// discriminant stored in `ptr` corresponds to a valid*
285	/// variant of that enum type, then it is guaranteed that the
286	/// appropriate bytes of `ptr` are initialized as defined by*
287	/// that variant's bit validity (although note that the
288	/// variant may contain another enum type, in which case the
289	/// same rules apply depending on the state of its
290	/// discriminant, and so on recursively).
291	AsInitialized < Initialized \| Valid,
292
293	/// The byte ranges in the referent are fully initialized. In
294	/// other words, if the referent is `N` bytes long, then it
295	/// contains a bit-valid `[u8; N]`.
296	Initialized,
297
298	/// The referent is bit-valid for `T`.
299	Valid,
300	}
301	}
302	}
303	}
304
305	pub(crate) use invariant::*;
306
307	/// External trait implementations on [`Ptr`].
308	mod _external {
309	use super::*;
310	use core::fmt::{Debug, Formatter};
311
312	/// SAFETY: Shared pointers are safely `Copy`. We do not implement `Copy`
313	/// for exclusive pointers, since at most one may exist at a time. `Ptr`'s
314	/// other invariants are unaffected by the number of references that exist
315	/// to `Ptr`'s referent.
316	impl<'a, T, I> Copy for Ptr<'a, T, I>
317	where
318	T: 'a + ?Sized,
319	I: Invariants,
320	Shared: AtLeast<I::Aliasing>,
321	{
322	}
323
324	/// SAFETY: Shared pointers are safely `Clone`. We do not implement `Clone`
325	/// for exclusive pointers, since at most one may exist at a time. `Ptr`'s
326	/// other invariants are unaffected by the number of references that exist
327	/// to `Ptr`'s referent.
328	impl<'a, T, I> Clone for Ptr<'a, T, I>
329	where
330	T: 'a + ?Sized,
331	I: Invariants,
332	Shared: AtLeast<I::Aliasing>,
333	{
334	#[inline]
335	fn clone(&self) -> Self {
336	*self
337	}
338	}
339
340	impl<'a, T, I> Debug for Ptr<'a, T, I>
341	where
342	T: 'a + ?Sized,
343	I: Invariants,
344	{
345	#[inline]
346	fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result {
347	self.as_non_null().fmt(f)
348	}
349	}
350	}
351
352	/// Methods for converting to and from `Ptr` and Rust's safe reference types.
353	mod _conversions {
354	use super::*;
355	use crate::util::{AlignmentVariance, Covariant, TransparentWrapper, ValidityVariance};
356
357	/// `&'a T` → `Ptr<'a, T>`
358	impl<'a, T> Ptr<'a, T, (Shared, Aligned, Valid)>
359	where
360	T: 'a + ?Sized,
361	{
362	/// Constructs a `Ptr` from a shared reference.
363	#[doc(hidden)]
364	#[inline]
365	pub fn from_ref(ptr: &'a T) -> Self {
366	let ptr = NonNull::from(ptr);
367	// SAFETY:
368	// 0. If `ptr`'s referent is not zero sized, then `ptr`, by
369	// invariant on `&'a T`, is derived from some valid Rust
370	// allocation, `A`.
371	// 1. If `ptr`'s referent is not zero sized, then `ptr`, by
372	// invariant on `&'a T`, has valid provenance for `A`.
373	// 2. If `ptr`'s referent is not zero sized, then `ptr`, by
374	// invariant on `&'a T`, addresses a byte range which is entirely
375	// contained in `A`.
376	// 3. `ptr`, by invariant on `&'a T`, addresses a byte range whose
377	// length fits in an `isize`.
378	// 4. `ptr`, by invariant on `&'a T`, addresses a byte range which
379	// does not wrap around the address space.
380	// 5. If `ptr`'s referent is not zero sized, then `A`, by invariant
381	// on `&'a T`, is guaranteed to live for at least `'a`.
382	// 6. `T: 'a`.
383	// 7. `ptr`, by invariant on `&'a T`, conforms to the aliasing
384	// invariant of `Shared`.
385	// 8. `ptr`, by invariant on `&'a T`, conforms to the alignment
386	// invariant of `Aligned`.
387	// 9. `ptr`, by invariant on `&'a T`, conforms to the validity
388	// invariant of `Valid`.
389	unsafe { Self::new(ptr) }
390	}
391	}
392
393	/// `&'a mut T` → `Ptr<'a, T>`
394	impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)>
395	where
396	T: 'a + ?Sized,
397	{
398	/// Constructs a `Ptr` from an exclusive reference.
399	#[inline]
400	pub(crate) fn from_mut(ptr: &'a mut T) -> Self {
401	let ptr = NonNull::from(ptr);
402	// SAFETY:
403	// 0. If `ptr`'s referent is not zero sized, then `ptr`, by
404	// invariant on `&'a mut T`, is derived from some valid Rust
405	// allocation, `A`.
406	// 1. If `ptr`'s referent is not zero sized, then `ptr`, by
407	// invariant on `&'a mut T`, has valid provenance for `A`.
408	// 2. If `ptr`'s referent is not zero sized, then `ptr`, by
409	// invariant on `&'a mut T`, addresses a byte range which is
410	// entirely contained in `A`.
411	// 3. `ptr`, by invariant on `&'a mut T`, addresses a byte range
412	// whose length fits in an `isize`.
413	// 4. `ptr`, by invariant on `&'a mut T`, addresses a byte range
414	// which does not wrap around the address space.
415	// 5. If `ptr`'s referent is not zero sized, then `A`, by invariant
416	// on `&'a mut T`, is guaranteed to live for at least `'a`.
417	// 6. `ptr`, by invariant on `&'a mut T`, conforms to the aliasing
418	// invariant of `Exclusive`.
419	// 7. `ptr`, by invariant on `&'a mut T`, conforms to the alignment
420	// invariant of `Aligned`.
421	// 8. `ptr`, by invariant on `&'a mut T`, conforms to the validity
422	// invariant of `Valid`.
423	unsafe { Self::new(ptr) }
424	}
425	}
426
427	/// `Ptr<'a, T>` → `&'a T`
428	impl<'a, T, I> Ptr<'a, T, I>
429	where
430	T: 'a + ?Sized,
431	I: Invariants<Alignment = Aligned, Validity = Valid>,
432	I::Aliasing: AtLeast<Shared>,
433	{
434	/// Converts `self` to a shared reference.
435	// This consumes `self`, not `&self`, because `self` is, logically, a
436	// pointer. For `I::Aliasing = invariant::Shared`, `Self: Copy`, and so
437	// this doesn't prevent the caller from still using the pointer after
438	// calling `as_ref`.
439	#[allow(clippy::wrong_self_convention)]
440	pub(crate) fn as_ref(self) -> &'a T {
441	let raw = self.as_non_null();
442	// SAFETY: This invocation of `NonNull::as_ref` satisfies its
443	// documented safety preconditions:
444	//
445	// 1. The pointer is properly aligned. This is ensured by-contract
446	// on `Ptr`, because the `I::Alignment` is `Aligned`.
447	//
448	// 2. If the pointer's referent is not zero-sized, then the pointer
449	// must be “dereferenceable” in the sense defined in the module
450	// documentation; i.e.:
451	//
452	// > The memory range of the given size starting at the pointer
453	// > must all be within the bounds of a single allocated object.
454	// > [2]
455	//
456	// This is ensured by contract on all `Ptr`s.
457	//
458	// 3. The pointer must point to an initialized instance of `T`. This
459	// is ensured by-contract on `Ptr`, because the `I::Validity` is
460	// `Valid`.
461	//
462	// 4. You must enforce Rust’s aliasing rules. This is ensured by
463	// contract on `Ptr`, because the `I::Aliasing` is
464	// `AtLeast<Shared>`. Either it is `Shared` or `Exclusive`. If it
465	// is `Shared`, other references may not mutate the referent
466	// outside of `UnsafeCell`s.
467	//
468	// [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_ref
469	// [2]: https://doc.rust-lang.org/std/ptr/index.html#safety
470	unsafe { raw.as_ref() }
471	}
472	}
473
474	impl<'a, T, I> Ptr<'a, T, I>
475	where
476	T: 'a + ?Sized,
477	I: Invariants,
478	I::Aliasing: AtLeast<Shared>,
479	{
480	/// Reborrows `self`, producing another `Ptr`.
481	///
482	/// Since `self` is borrowed immutably, this prevents any mutable
483	/// methods from being called on `self` as long as the returned `Ptr`
484	/// exists.
485	#[doc(hidden)]
486	#[inline]
487	#[allow(clippy::needless_lifetimes)] // Allows us to name the lifetime in the safety comment below.
488	pub fn reborrow<'b>(&'b mut self) -> Ptr<'b, T, I>
489	where
490	'a: 'b,
491	{
492	// SAFETY: The following all hold by invariant on `self`, and thus
493	// hold of `ptr = self.as_non_null()`:
494	// 0. If `ptr`'s referent is not zero sized, then `ptr` is derived
495	// from some valid Rust allocation, `A`.
496	// 1. If `ptr`'s referent is not zero sized, then `ptr` has valid
497	// provenance for `A`.
498	// 2. If `ptr`'s referent is not zero sized, then `ptr` addresses a
499	// byte range which is entirely contained in `A`.
500	// 3. `ptr` addresses a byte range whose length fits in an `isize`.
501	// 4. `ptr` addresses a byte range which does not wrap around the
502	// address space.
503	// 5. If `ptr`'s referent is not zero sized, then `A` is guaranteed
504	// to live for at least `'a`.
505	// 6. SEE BELOW.
506	// 7. `ptr` conforms to the alignment invariant of
507	// [`I::Alignment`](invariant::Alignment).
508	// 8. `ptr` conforms to the validity invariant of
509	// [`I::Validity`](invariant::Validity).
510	//
511	// For aliasing (6 above), since `I::Aliasing: AtLeast<Shared>`,
512	// there are two cases for `I::Aliasing`:
513	// - For `invariant::Shared`: `'a` outlives `'b`, and so the
514	// returned `Ptr` does not permit accessing the referent any
515	// longer than is possible via `self`. For shared aliasing, it is
516	// sound for multiple `Ptr`s to exist simultaneously which
517	// reference the same memory, so creating a new one is not
518	// problematic.
519	// - For `invariant::Exclusive`: Since `self` is `&'b mut` and we
520	// return a `Ptr` with lifetime `'b`, `self` is inaccessible to
521	// the caller for the lifetime `'b` - in other words, `self` is
522	// inaccessible to the caller as long as the returned `Ptr`
523	// exists. Since `self` is an exclusive `Ptr`, no other live
524	// references or `Ptr`s may exist which refer to the same memory
525	// while `self` is live. Thus, as long as the returned `Ptr`
526	// exists, no other references or `Ptr`s which refer to the same
527	// memory may be live.
528	unsafe { Ptr::new(self.as_non_null()) }
529	}
530	}
531
532	/// `Ptr<'a, T>` → `&'a mut T`
533	impl<'a, T> Ptr<'a, T, (Exclusive, Aligned, Valid)>
534	where
535	T: 'a + ?Sized,
536	{
537	/// Converts `self` to a mutable reference.
538	#[allow(clippy::wrong_self_convention)]
539	pub(crate) fn as_mut(self) -> &'a mut T {
540	let mut raw = self.as_non_null();
541	// SAFETY: This invocation of `NonNull::as_mut` satisfies its
542	// documented safety preconditions:
543	//
544	// 1. The pointer is properly aligned. This is ensured by-contract
545	// on `Ptr`, because the `ALIGNMENT_INVARIANT` is `Aligned`.
546	//
547	// 2. If the pointer's referent is not zero-sized, then the pointer
548	// must be “dereferenceable” in the sense defined in the module
549	// documentation; i.e.:
550	//
551	// > The memory range of the given size starting at the pointer
552	// > must all be within the bounds of a single allocated object.
553	// > [2]
554	//
555	// This is ensured by contract on all `Ptr`s.
556	//
557	// 3. The pointer must point to an initialized instance of `T`. This
558	// is ensured by-contract on `Ptr`, because the
559	// `VALIDITY_INVARIANT` is `Valid`.
560	//
561	// 4. You must enforce Rust’s aliasing rules. This is ensured by
562	// contract on `Ptr`, because the `ALIASING_INVARIANT` is
563	// `Exclusive`.
564	//
565	// [1]: https://doc.rust-lang.org/std/ptr/struct.NonNull.html#method.as_mut
566	// [2]: https://doc.rust-lang.org/std/ptr/index.html#safety
567	unsafe { raw.as_mut() }
568	}
569	}
570
571	/// `Ptr<'a, T = Wrapper<U>>` → `Ptr<'a, U>`
572	impl<'a, T, I> Ptr<'a, T, I>
573	where
574	T: 'a + TransparentWrapper<I, UnsafeCellVariance = Covariant> + ?Sized,
575	I: Invariants,
576	{
577	/// Converts `self` to a transparent wrapper type into a `Ptr` to the
578	/// wrapped inner type.
579	pub(crate) fn transparent_wrapper_into_inner(
580	self,
581	) -> Ptr<
582	'a,
583	T::Inner,
584	(
585	I::Aliasing,
586	<T::AlignmentVariance as AlignmentVariance<I::Alignment>>::Applied,
587	<T::ValidityVariance as ValidityVariance<I::Validity>>::Applied,
588	),
589	> {
590	// SAFETY:
591	// - By invariant on `TransparentWrapper::cast_into_inner`:
592	// - This cast preserves address and referent size, and thus the
593	// returned pointer addresses the same bytes as `p`
594	// - This cast preserves provenance
595	// - By invariant on `TransparentWrapper<UnsafeCellVariance =
596	// Covariant>`, `T` and `T::Inner` have `UnsafeCell`s at the same
597	// byte ranges. Since `p` and the returned pointer address the
598	// same byte range, they refer to `UnsafeCell`s at the same byte
599	// ranges.
600	let c = unsafe { self.cast_unsized(\|p\| T::cast_into_inner(p)) };
601	// SAFETY: By invariant on `TransparentWrapper`, since `self`
602	// satisfies the alignment invariant `I::Alignment`, `c` (of type
603	// `T::Inner`) satisfies the given "applied" alignment invariant.
604	let c = unsafe {
605	c.assume_alignment::<<T::AlignmentVariance as AlignmentVariance<I::Alignment>>::Applied>()
606	};
607	// SAFETY: By invariant on `TransparentWrapper`, since `self`
608	// satisfies the validity invariant `I::Validity`, `c` (of type
609	// `T::Inner`) satisfies the given "applied" validity invariant.
610	let c = unsafe {
611	c.assume_validity::<<T::ValidityVariance as ValidityVariance<I::Validity>>::Applied>()
612	};
613	c
614	}
615	}
616
617	/// `Ptr<'a, T, (_, _, _)>` → `Ptr<'a, Unalign<T>, (_, Aligned, _)>`
618	impl<'a, T, I> Ptr<'a, T, I>
619	where
620	I: Invariants,
621	{
622	/// Converts a `Ptr` an unaligned `T` into a `Ptr` to an aligned
623	/// `Unalign<T>`.
624	pub(crate) fn into_unalign(
625	self,
626	) -> Ptr<'a, crate::Unalign<T>, (I::Aliasing, Aligned, I::Validity)> {
627	// SAFETY:
628	// - This cast preserves provenance.
629	// - This cast preserves address. `Unalign<T>` promises to have the
630	// same size as `T`, and so the cast returns a pointer addressing
631	// the same byte range as `p`.
632	// - By the same argument, the returned pointer refers to
633	// `UnsafeCell`s at the same locations as `p`.
634	let ptr = unsafe {
635	#[allow(clippy::as_conversions)]
636	self.cast_unsized(\|p: *mut T\| p as *mut crate::Unalign<T>)
637	};
638	// SAFETY: `Unalign<T>` promises to have the same bit validity as
639	// `T`.
640	let ptr = unsafe { ptr.assume_validity::<I::Validity>() };
641	// SAFETY: `Unalign<T>` promises to have alignment 1, and so it is
642	// trivially aligned.
643	let ptr = unsafe { ptr.assume_alignment::<Aligned>() };
644	ptr
645	}
646	}
647	}
648
649	/// State transitions between invariants.
650	mod _transitions {
651	use super::*;
652	use crate::{AlignmentError, TryFromBytes, ValidityError};
653
654	impl<'a, T, I> Ptr<'a, T, I>
655	where
656	T: 'a + ?Sized,
657	I: Invariants,
658	{
659	/// Returns a `Ptr` with [`Exclusive`] aliasing if `self` already has
660	/// `Exclusive` aliasing.
661	///
662	/// This allows code which is generic over aliasing to down-cast to a
663	/// concrete aliasing.
664	///
665	/// [`Exclusive`]: invariant::Exclusive
666	#[inline]
667	pub(crate) fn into_exclusive_or_post_monomorphization_error(
668	self,
669	) -> Ptr<'a, T, (Exclusive, I::Alignment, I::Validity)> {
670	trait AliasingExt: Aliasing {
671	const IS_EXCLUSIVE: bool;
672	}
673
674	impl<A: Aliasing> AliasingExt for A {
675	const IS_EXCLUSIVE: bool = {
676	let is_exclusive =
677	strs_are_equal(<Self as Aliasing>::NAME, <Exclusive as Aliasing>::NAME);
678	const_assert!(is_exclusive);
679	`true`
680	};
681	}
682
683	const fn strs_are_equal(s: &str, t: &str) -> bool {
684	if s.len() != t.len() {
685	return `false`;
686	}
687
688	let s = s.as_bytes();
689	let t = t.as_bytes();
690
691	let mut i = `0`;
692	#[allow(clippy::arithmetic_side_effects)]
693	while i < s.len() {
694	#[allow(clippy::indexing_slicing)]
695	if s[i] != t[i] {
696	return `false`;
697	}
698
699	i += `1`;
700	}
701
702	`true`
703	}
704
705	assert!(I::Aliasing::IS_EXCLUSIVE);
706
707	// SAFETY: We've confirmed that `self` already has the aliasing
708	// `Exclusive`. If it didn't, either the preceding assert would fail
709	// or evaluating `I::Aliasing::IS_EXCLUSIVE` would fail. We're
710	// pretty* sure that it's guaranteed to fail const eval, but the*
711	// `assert!` provides a backstop in case that doesn't work.
712	unsafe { self.assume_exclusive() }
713	}
714
715	/// Assumes that `self` satisfies the invariants `H`.
716	///
717	/// # Safety
718	///
719	/// The caller promises that `self` satisfies the invariants `H`.
720	unsafe fn assume_invariants<H: Invariants>(self) -> Ptr<'a, T, H> {
721	// SAFETY: The caller has promised to satisfy all parameterized
722	// invariants of `Ptr`. `Ptr`'s other invariants are satisfied
723	// by-contract by the source `Ptr`.
724	unsafe { Ptr::new(self.as_non_null()) }
725	}
726
727	/// Helps the type system unify two distinct invariant types which are
728	/// actually the same.
729	pub(crate) fn unify_invariants<
730	H: Invariants<Aliasing = I::Aliasing, Alignment = I::Alignment, Validity = I::Validity>,
731	>(
732	self,
733	) -> Ptr<'a, T, H> {
734	// SAFETY: The associated type bounds on `H` ensure that the
735	// invariants are unchanged.
736	unsafe { self.assume_invariants::<H>() }
737	}
738
739	/// Assumes that `self` satisfies the aliasing requirement of `A`.
740	///
741	/// # Safety
742	///
743	/// The caller promises that `self` satisfies the aliasing requirement
744	/// of `A`.
745	#[inline]
746	pub(crate) unsafe fn assume_aliasing<A: Aliasing>(
747	self,
748	) -> Ptr<'a, T, (A, I::Alignment, I::Validity)> {
749	// SAFETY: The caller promises that `self` satisfies the aliasing
750	// requirements of `A`.
751	unsafe { self.assume_invariants() }
752	}
753
754	/// Assumes `self` satisfies the aliasing requirement of [`Exclusive`].
755	///
756	/// # Safety
757	///
758	/// The caller promises that `self` satisfies the aliasing requirement
759	/// of `Exclusive`.
760	///
761	/// [`Exclusive`]: invariant::Exclusive
762	#[inline]
763	pub(crate) unsafe fn assume_exclusive(
764	self,
765	) -> Ptr<'a, T, (Exclusive, I::Alignment, I::Validity)> {
766	// SAFETY: The caller promises that `self` satisfies the aliasing
767	// requirements of `Exclusive`.
768	unsafe { self.assume_aliasing::<Exclusive>() }
769	}
770
771	/// Assumes that `self`'s referent is validly-aligned for `T` if
772	/// required by `A`.
773	///
774	/// # Safety
775	///
776	/// The caller promises that `self`'s referent conforms to the alignment
777	/// invariant of `T` if required by `A`.
778	#[inline]
779	pub(crate) unsafe fn assume_alignment<A: Alignment>(
780	self,
781	) -> Ptr<'a, T, (I::Aliasing, A, I::Validity)> {
782	// SAFETY: The caller promises that `self`'s referent is
783	// well-aligned for `T` if required by `A` .
784	unsafe { self.assume_invariants() }
785	}
786
787	/// Checks the `self`'s alignment at runtime, returning an aligned `Ptr`
788	/// on success.
789	pub(crate) fn bikeshed_try_into_aligned(
790	self,
791	) -> Result<Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)>, AlignmentError<Self, T>>
792	where
793	T: Sized,
794	{
795	if let Err(err) = crate::util::validate_aligned_to::<_, T>(self.as_non_null()) {
796	return Err(err.with_src(self));
797	}
798
799	// SAFETY: We just checked the alignment.
800	Ok(unsafe { self.assume_alignment::<Aligned>() })
801	}
802
803	/// Recalls that `self`'s referent is validly-aligned for `T`.
804	#[inline]
805	// TODO(#859): Reconsider the name of this method before making it
806	// public.
807	pub(crate) fn bikeshed_recall_aligned(
808	self,
809	) -> Ptr<'a, T, (I::Aliasing, Aligned, I::Validity)>
810	where
811	T: crate::Unaligned,
812	{
813	// SAFETY: The bound `T: Unaligned` ensures that `T` has no
814	// non-trivial alignment requirement.
815	unsafe { self.assume_alignment::<Aligned>() }
816	}
817
818	/// Assumes that `self`'s referent conforms to the validity requirement
819	/// of `V`.
820	///
821	/// # Safety
822	///
823	/// The caller promises that `self`'s referent conforms to the validity
824	/// requirement of `V`.
825	#[doc(hidden)]
826	#[must_use]
827	#[inline]
828	pub unsafe fn assume_validity<V: Validity>(
829	self,
830	) -> Ptr<'a, T, (I::Aliasing, I::Alignment, V)> {
831	// SAFETY: The caller promises that `self`'s referent conforms to
832	// the validity requirement of `V`.
833	unsafe { self.assume_invariants() }
834	}
835
836	/// A shorthand for `self.assume_validity<invariant::Initialized>()`.
837	///
838	/// # Safety
839	///
840	/// The caller promises to uphold the safety preconditions of
841	/// `self.assume_validity<invariant::Initialized>()`.
842	#[doc(hidden)]
843	#[must_use]
844	#[inline]
845	pub unsafe fn assume_initialized(
846	self,
847	) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Initialized)> {
848	// SAFETY: The caller has promised to uphold the safety
849	// preconditions.
850	unsafe { self.assume_validity::<Initialized>() }
851	}
852
853	/// A shorthand for `self.assume_validity<Valid>()`.
854	///
855	/// # Safety
856	///
857	/// The caller promises to uphold the safety preconditions of
858	/// `self.assume_validity<Valid>()`.
859	#[doc(hidden)]
860	#[must_use]
861	#[inline]
862	pub unsafe fn assume_valid(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)> {
863	// SAFETY: The caller has promised to uphold the safety
864	// preconditions.
865	unsafe { self.assume_validity::<Valid>() }
866	}
867
868	/// Recalls that `self`'s referent is bit-valid for `T`.
869	#[doc(hidden)]
870	#[must_use]
871	#[inline]
872	// TODO(#859): Reconsider the name of this method before making it
873	// public.
874	pub fn bikeshed_recall_valid(self) -> Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)>
875	where
876	T: crate::FromBytes,
877	I: Invariants<Validity = Initialized>,
878	{
879	// SAFETY: The bound `T: FromBytes` ensures that any initialized
880	// sequence of bytes is bit-valid for `T`. `I: Invariants<Validity =
881	// invariant::Initialized>` ensures that all of the referent bytes
882	// are initialized.
883	unsafe { self.assume_valid() }
884	}
885
886	/// Checks that `self`'s referent is validly initialized for `T`,
887	/// returning a `Ptr` with `Valid` on success.
888	///
889	/// # Panics
890	///
891	/// This method will panic if
892	/// [`T::is_bit_valid`][TryFromBytes::is_bit_valid] panics.
893	///
894	/// # Safety
895	///
896	/// On error, unsafe code may rely on this method's returned
897	/// `ValidityError` containing `self`.
898	#[inline]
899	pub(crate) fn try_into_valid(
900	mut self,
901	) -> Result<Ptr<'a, T, (I::Aliasing, I::Alignment, Valid)>, ValidityError<Self, T>>
902	where
903	T: TryFromBytes,
904	I::Aliasing: AtLeast<Shared>,
905	I: Invariants<Validity = Initialized>,
906	{
907	// This call may panic. If that happens, it doesn't cause any soundness
908	// issues, as we have not generated any invalid state which we need to
909	// fix before returning.
910	if T::is_bit_valid(self.reborrow().forget_aligned()) {
911	// SAFETY: If `T::is_bit_valid`, code may assume that `self`
912	// contains a bit-valid instance of `Self`.
913	Ok(unsafe { self.assume_valid() })
914	} else {
915	Err(ValidityError::new(self))
916	}
917	}
918
919	/// Forgets that `self`'s referent exclusively references `T`,
920	/// downgrading to a shared reference.
921	#[doc(hidden)]
922	#[must_use]
923	#[inline]
924	pub fn forget_exclusive(self) -> Ptr<'a, T, (Shared, I::Alignment, I::Validity)>
925	where
926	I::Aliasing: AtLeast<Shared>,
927	{
928	// SAFETY: `I::Aliasing` is at least as restrictive as `Shared`.
929	unsafe { self.assume_invariants() }
930	}
931
932	/// Forgets that `self`'s referent is validly-aligned for `T`.
933	#[doc(hidden)]
934	#[must_use]
935	#[inline]
936	pub fn forget_aligned(self) -> Ptr<'a, T, (I::Aliasing, Any, I::Validity)> {
937	// SAFETY: `Any` is less restrictive than `Aligned`.
938	unsafe { self.assume_invariants() }
939	}
940	}
941	}
942
943	/// Casts of the referent type.
944	mod _casts {
945	use super::*;
946	use crate::{
947	layout::{DstLayout, MetadataCastError},
948	pointer::aliasing_safety::*,
949	AlignmentError, CastError, PointerMetadata, SizeError,
950	};
951
952	impl<'a, T, I> Ptr<'a, T, I>
953	where
954	T: 'a + ?Sized,
955	I: Invariants,
956	{
957	/// Casts to a different (unsized) target type.
958	///
959	/// # Safety
960	///
961	/// The caller promises that `u = cast(p)` is a pointer cast with the
962	/// following properties:
963	/// - `u` addresses a subset of the bytes addressed by `p`
964	/// - `u` has the same provenance as `p`
965	/// - If `I::Aliasing` is [`Any`] or [`Shared`], `UnsafeCell`s in `u`*
966	/// must exist at ranges identical to those at which `UnsafeCell`s
967	/// exist in `p`*
968	#[doc(hidden)]
969	#[inline]
970	pub unsafe fn cast_unsized<U: 'a + ?Sized, F: FnOnce(*mut T) -> *mut U>(
971	self,
972	cast: F,
973	) -> Ptr<'a, U, (I::Aliasing, Any, Any)> {
974	let ptr = cast(self.as_non_null().as_ptr());
975
976	// SAFETY: Caller promises that `cast` returns a pointer whose
977	// address is in the range of `self.as_non_null()`'s referent. By
978	// invariant, none of these addresses are null.
979	let ptr = unsafe { NonNull::new_unchecked(ptr) };
980
981	// SAFETY:
982	//
983	// Lemma 1: `ptr` has the same provenance as `self`. The caller
984	// promises that `cast` preserves provenance, and we call it with
985	// `self.as_non_null()`.
986	//
987	// 0. By invariant, if `self`'s referent is not zero sized, then
988	// `self` is derived from some valid Rust allocation, `A`. By
989	// Lemma 1, `ptr` has the same provenance as `self`. Thus, `ptr`
990	// is derived from `A`.
991	// 1. By invariant, if `self`'s referent is not zero sized, then
992	// `self` has valid provenance for `A`. By Lemma 1, so does
993	// `ptr`.
994	// 2. By invariant on `self` and caller precondition, if `ptr`'s
995	// referent is not zero sized, then `ptr` addresses a byte range
996	// which is entirely contained in `A`.
997	// 3. By invariant on `self` and caller precondition, `ptr`
998	// addresses a byte range whose length fits in an `isize`.
999	// 4. By invariant on `self` and caller precondition, `ptr`
1000	// addresses a byte range which does not wrap around the address
1001	// space.
1002	// 5. By invariant on `self`, if `self`'s referent is not zero
1003	// sized, then `A` is guaranteed to live for at least `'a`.
1004	// 6. `ptr` conforms to the aliasing invariant of `I::Aliasing`:
1005	// - `Exclusive`: `self` is the only `Ptr` or reference which is
1006	// permitted to read or modify the referent for the lifetime
1007	// `'a`. Since we consume `self` by value, the returned pointer
1008	// remains the only `Ptr` or reference which is permitted to
1009	// read or modify the referent for the lifetime `'a`.
1010	// - `Shared`: Since `self` has aliasing `Shared`, we know that
1011	// no other code may mutate the referent during the lifetime
1012	// `'a`, except via `UnsafeCell`s. The caller promises that
1013	// `UnsafeCell`s cover the same byte ranges in `self` and*
1014	// `ptr`. For each byte in the referent, there are two cases:*
1015	// - If the byte is not covered by an `UnsafeCell` in `ptr`,*
1016	// then it is not covered in `self`. By invariant on `self`,*
1017	// it will not be mutated during `'a`, as required by the
1018	// constructed pointer. Similarly, the returned pointer will
1019	// not permit any mutations to these locations, as required
1020	// by the invariant on `self`.
1021	// - If the byte is covered by an `UnsafeCell` in `ptr`, then*
1022	// the returned pointer's invariants do not assume that the
1023	// byte will not be mutated during `'a`. While the returned
1024	// pointer will permit mutation of this byte during `'a`, by
1025	// invariant on `self`, no other code assumes that this will
1026	// not happen.
1027	// 7. `ptr`, trivially, conforms to the alignment invariant of
1028	// `Any`.
1029	// 8. `ptr`, trivially, conforms to the validity invariant of `Any`.
1030	unsafe { Ptr::new(ptr) }
1031	}
1032	}
1033
1034	impl<'a, T, I> Ptr<'a, T, I>
1035	where
1036	T: 'a + KnownLayout + ?Sized,
1037	I: Invariants<Validity = Initialized>,
1038	{
1039	/// Casts this pointer-to-initialized into a pointer-to-bytes.
1040	#[allow(clippy::wrong_self_convention)]
1041	pub(crate) fn as_bytes<R>(self) -> Ptr<'a, [u8], (I::Aliasing, Aligned, Valid)>
1042	where
1043	[u8]: AliasingSafe<T, I::Aliasing, R>,
1044	R: AliasingSafeReason,
1045	{
1046	let bytes = match T::size_of_val_raw(self.as_non_null()) {
1047	Some(bytes) => bytes,
1048	// SAFETY: `KnownLayout::size_of_val_raw` promises to always
1049	// return `Some` so long as the resulting size fits in a
1050	// `usize`. By invariant on `Ptr`, `self` refers to a range of
1051	// bytes whose size fits in an `isize`, which implies that it
1052	// also fits in a `usize`.
1053	None => unsafe { core::hint::unreachable_unchecked() },
1054	};
1055
1056	// SAFETY:
1057	// - `slice_from_raw_parts_mut` and `.cast` both preserve the
1058	// pointer's address, and `bytes` is the length of `p`, so the
1059	// returned pointer addresses the same bytes as `p`
1060	// - `slice_from_raw_parts_mut` and `.cast` both preserve provenance
1061	// - Because `[u8]: AliasingSafe<T, I::Aliasing, _>`, either:
1062	// - `I::Aliasing` is `Exclusive`
1063	// - `T` and `[u8]` are both `Immutable`, in which case they
1064	// trivially contain `UnsafeCell`s at identical locations
1065	let ptr: Ptr<'a, [u8], _> = unsafe {
1066	self.cast_unsized(\|p: *mut T\| {
1067	#[allow(clippy::as_conversions)]
1068	core::ptr::slice_from_raw_parts_mut(p.cast::<u8>(), bytes)
1069	})
1070	};
1071
1072	let ptr = ptr.bikeshed_recall_aligned();
1073
1074	// SAFETY: `ptr`'s referent begins as `Initialized`, denoting that
1075	// all bytes of the referent are initialized bytes. The referent
1076	// type is then casted to `[u8]`, whose only validity invariant is
1077	// that its bytes are initialized. This validity invariant is
1078	// satisfied by the `Initialized` invariant on the starting `ptr`.
1079	unsafe { ptr.assume_validity::<Valid>() }
1080	}
1081	}
1082
1083	impl<'a, T, I, const N: usize> Ptr<'a, [T; N], I>
1084	where
1085	T: 'a,
1086	I: Invariants,
1087	{
1088	/// Casts this pointer-to-array into a slice.
1089	#[allow(clippy::wrong_self_convention)]
1090	pub(crate) fn as_slice(self) -> Ptr<'a, [T], I> {
1091	let start = self.as_non_null().cast::<T>().as_ptr();
1092	let slice = core::ptr::slice_from_raw_parts_mut(start, N);
1093	// SAFETY: `slice` is not null, because it is derived from `start`
1094	// which is non-null.
1095	let slice = unsafe { NonNull::new_unchecked(slice) };
1096	// SAFETY: Lemma: In the following safety arguments, note that
1097	// `slice` is derived from `self` in two steps: first, by casting
1098	// `self: [T; N]` to `start: T`, then by constructing a pointer to a
1099	// slice starting at `start` of length `N`. As a result, `slice`
1100	// references exactly the same allocation as `self`, if any.
1101	//
1102	// 0. By the above lemma, if `slice`'s referent is not zero sized,
1103	// then `slice` is derived from the same allocation as `self`,
1104	// which, by invariant on `Ptr`, is valid.
1105	// 1. By the above lemma, if `slice`'s referent is not zero sized,
1106	// then , `slice` has valid provenance for `A`, since it is
1107	// derived from the pointer `self`, which, by invariant on `Ptr`,
1108	// has valid provenance for `A`.
1109	// 2. By the above lemma, if `slice`'s referent is not zero sized,
1110	// then `slice` addresses a byte range which is entirely
1111	// contained in `A`, because it references exactly the same byte
1112	// range as `self`, which, by invariant on `Ptr`, is entirely
1113	// contained in `A`.
1114	// 3. By the above lemma, `slice` addresses a byte range whose
1115	// length fits in an `isize`, since it addresses exactly the same
1116	// byte range as `self`, which, by invariant on `Ptr`, has a
1117	// length that fits in an `isize`.
1118	// 4. By the above lemma, `slice` addresses a byte range which does
1119	// not wrap around the address space, since it addresses exactly
1120	// the same byte range as `self`, which, by invariant on `Ptr`,
1121	// does not wrap around the address space.
1122	// 5. By the above lemma, if `slice`'s referent is not zero sized,
1123	// then `A` is guaranteed to live for at least `'a`, because it
1124	// is derived from the same allocation as `self`, which, by
1125	// invariant on `Ptr`, lives for at least `'a`.
1126	// 6. By the above lemma, `slice` conforms to the aliasing invariant
1127	// of `I::Aliasing`, because the operations that produced `slice`
1128	// from `self` do not impact aliasing.
1129	// 7. By the above lemma, `slice` conforms to the alignment
1130	// invariant of `I::Alignment`, because the operations that
1131	// produced `slice` from `self` do not impact alignment.
1132	// 8. By the above lemma, `slice` conforms to the validity invariant
1133	// of `I::Validity`, because the operations that produced `slice`
1134	// from `self` do not impact validity.
1135	unsafe { Ptr::new(slice) }
1136	}
1137	}
1138
1139	/// For caller convenience, these methods are generic over alignment
1140	/// invariant. In practice, the referent is always well-aligned, because the
1141	/// alignment of `[u8]` is 1.
1142	impl<'a, I> Ptr<'a, [u8], I>
1143	where
1144	I: Invariants<Validity = Valid>,
1145	{
1146	/// Attempts to cast `self` to a `U` using the given cast type.
1147	///
1148	/// If `U` is a slice DST and pointer metadata (`meta`) is provided,
1149	/// then the cast will only succeed if it would produce an object with
1150	/// the given metadata.
1151	///
1152	/// Returns `None` if the resulting `U` would be invalidly-aligned, if
1153	/// no `U` can fit in `self`, or if the provided pointer metadata
1154	/// describes an invalid instance of `U`. On success, returns a pointer
1155	/// to the largest-possible `U` which fits in `self`.
1156	///
1157	/// # Safety
1158	///
1159	/// The caller may assume that this implementation is correct, and may
1160	/// rely on that assumption for the soundness of their code. In
1161	/// particular, the caller may assume that, if `try_cast_into` returns
1162	/// `Some((ptr, remainder))`, then `ptr` and `remainder` refer to
1163	/// non-overlapping byte ranges within `self`, and that `ptr` and
1164	/// `remainder` entirely cover `self`. Finally:
1165	/// - If this is a prefix cast, `ptr` has the same address as `self`.
1166	/// - If this is a suffix cast, `remainder` has the same address as
1167	/// `self`.
1168	#[inline(always)]
1169	pub(crate) fn try_cast_into<U, R>(
1170	self,
1171	cast_type: CastType,
1172	meta: Option<U::PointerMetadata>,
1173	) -> Result<
1174	(Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, Ptr<'a, [u8], I>),
1175	CastError<Self, U>,
1176	>
1177	where
1178	R: AliasingSafeReason,
1179	U: 'a + ?Sized + KnownLayout + AliasingSafe<[u8], I::Aliasing, R>,
1180	{
1181	let layout = match meta {
1182	None => U::LAYOUT,
1183	// This can return `None` if the metadata describes an object
1184	// which can't fit in an `isize`.
1185	Some(meta) => {
1186	let size = match meta.size_for_metadata(U::LAYOUT) {
1187	Some(size) => size,
1188	None => return Err(CastError::Size(SizeError::new(self))),
1189	};
1190	DstLayout { align: U::LAYOUT.align, size_info: crate::SizeInfo::Sized { size } }
1191	}
1192	};
1193	// PANICS: By invariant, the byte range addressed by `self.ptr` does
1194	// not wrap around the address space. This implies that the sum of
1195	// the address (represented as a `usize`) and length do not overflow
1196	// `usize`, as required by `validate_cast_and_convert_metadata`.
1197	// Thus, this call to `validate_cast_and_convert_metadata` will only
1198	// panic if `U` is a DST whose trailing slice element is zero-sized.
1199	let maybe_metadata = layout.validate_cast_and_convert_metadata(
1200	AsAddress::addr(self.as_non_null().as_ptr()),
1201	self.len(),
1202	cast_type,
1203	);
1204
1205	let (elems, split_at) = match maybe_metadata {
1206	Ok((elems, split_at)) => (elems, split_at),
1207	Err(MetadataCastError::Alignment) => {
1208	// SAFETY: Since `validate_cast_and_convert_metadata`
1209	// returned an alignment error, `U` must have an alignment
1210	// requirement greater than one.
1211	let err = unsafe { AlignmentError::<_, U>::new_unchecked(self) };
1212	return Err(CastError::Alignment(err));
1213	}
1214	Err(MetadataCastError::Size) => return Err(CastError::Size(SizeError::new(self))),
1215	};
1216
1217	// SAFETY: `validate_cast_and_convert_metadata` promises to return
1218	// `split_at <= self.len()`.
1219	let (l_slice, r_slice) = unsafe { self.split_at(split_at) };
1220
1221	let (target, remainder) = match cast_type {
1222	CastType::Prefix => (l_slice, r_slice),
1223	CastType::Suffix => (r_slice, l_slice),
1224	};
1225
1226	let base = target.as_non_null().cast::<u8>();
1227
1228	let elems = <U as KnownLayout>::PointerMetadata::from_elem_count(elems);
1229	// For a slice DST type, if `meta` is `Some(elems)`, then we
1230	// synthesize `layout` to describe a sized type whose size is equal
1231	// to the size of the instance that we are asked to cast. For sized
1232	// types, `validate_cast_and_convert_metadata` returns `elems == 0`.
1233	// Thus, in this case, we need to use the `elems` passed by the
1234	// caller, not the one returned by
1235	// `validate_cast_and_convert_metadata`.
1236	let elems = meta.unwrap_or(elems);
1237
1238	let ptr = U::raw_from_ptr_len(base, elems);
1239
1240	// SAFETY:
1241	// 0. By invariant, if `target`'s referent is not zero sized, then
1242	// `target` is derived from some valid Rust allocation, `A`. By
1243	// contract on `cast`, `ptr` is derived from `self`, and thus
1244	// from the same valid Rust allocation, `A`.
1245	// 1. By invariant, if `target`'s referent is not zero sized, then
1246	// `target` has provenance valid for some Rust allocation, `A`.
1247	// Because `ptr` is derived from `target` via
1248	// provenance-preserving operations, `ptr` will also have
1249	// provenance valid for `A`.
1250	// - `validate_cast_and_convert_metadata` promises that the object
1251	// described by `elems` and `split_at` lives at a byte range
1252	// which is a subset of the input byte range. Thus:
1253	// 2. Since, by invariant, if `target`'s referent is not zero
1254	// sized, then `target` addresses a byte range which is
1255	// entirely contained in `A`, so does `ptr`.
1256	// 3. Since, by invariant, `target` addresses a byte range whose
1257	// length fits in an `isize`, so does `ptr`.
1258	// 4. Since, by invariant, `target` addresses a byte range which
1259	// does not wrap around the address space, so does `ptr`.
1260	// 5. Since, by invariant, if `target`'s referent is not zero
1261	// sized, then `target` refers to an allocation which is
1262	// guaranteed to live for at least `'a`, so does `ptr`.
1263	// 6. Since `U: AliasingSafe<[u8], I::Aliasing, _>`, either:
1264	// - `I::Aliasing` is `Exclusive`, in which case both `src`
1265	// and `ptr` conform to `Exclusive`
1266	// - `I::Aliasing` is `Shared` or `Any` and both `U` and
1267	// `[u8]` are `Immutable`. In this case, neither pointer
1268	// permits mutation, and so `Shared` aliasing is satisfied.
1269	// 7. `ptr` conforms to the alignment invariant of `Aligned` because
1270	// it is derived from `validate_cast_and_convert_metadata`, which
1271	// promises that the object described by `target` is validly
1272	// aligned for `U`.
1273	// 8. By trait bound, `self` - and thus `target` - is a bit-valid
1274	// `[u8]`. All bit-valid `[u8]`s have all of their bytes
1275	// initialized, so `ptr` conforms to the validity invariant of
1276	// `Initialized`.
1277	Ok((unsafe { Ptr::new(ptr) }, remainder))
1278	}
1279
1280	/// Attempts to cast `self` into a `U`, failing if all of the bytes of
1281	/// `self` cannot be treated as a `U`.
1282	///
1283	/// In particular, this method fails if `self` is not validly-aligned
1284	/// for `U` or if `self`'s size is not a valid size for `U`.
1285	///
1286	/// # Safety
1287	///
1288	/// On success, the caller may assume that the returned pointer
1289	/// references the same byte range as `self`.
1290	#[allow(unused)]
1291	#[inline(always)]
1292	pub(crate) fn try_cast_into_no_leftover<U, R>(
1293	self,
1294	meta: Option<U::PointerMetadata>,
1295	) -> Result<Ptr<'a, U, (I::Aliasing, Aligned, Initialized)>, CastError<Self, U>>
1296	where
1297	U: 'a + ?Sized + KnownLayout + AliasingSafe<[u8], I::Aliasing, R>,
1298	R: AliasingSafeReason,
1299	{
1300	// TODO(#67): Remove this allow. See NonNulSlicelExt for more
1301	// details.
1302	#[allow(unstable_name_collisions)]
1303	match self.try_cast_into(CastType::Prefix, meta) {
1304	Ok((slf, remainder)) => {
1305	if remainder.len() == `0` {
1306	Ok(slf)
1307	} else {
1308	// Undo the cast so we can return the original bytes.
1309	let slf = slf.as_bytes();
1310	// Restore the initial alignment invariant of `self`.
1311	//
1312	// SAFETY: The referent type of `slf` is now equal to
1313	// that of `self`, but the alignment invariants
1314	// nominally differ. Since `slf` and `self` refer to the
1315	// same memory and no actions have been taken that would
1316	// violate the original invariants on `self`, it is
1317	// sound to apply the alignment invariant of `self` onto
1318	// `slf`.
1319	let slf = unsafe { slf.assume_alignment::<I::Alignment>() };
1320	let slf = slf.unify_invariants();
1321	Err(CastError::Size(SizeError::<_, U>::new(slf)))
1322	}
1323	}
1324	Err(err) => Err(err),
1325	}
1326	}
1327	}
1328
1329	impl<'a, T, I> Ptr<'a, core::cell::UnsafeCell<T>, I>
1330	where
1331	T: 'a + ?Sized,
1332	I: Invariants<Aliasing = Exclusive>,
1333	{
1334	/// Converts this `Ptr` into a pointer to the underlying data.
1335	///
1336	/// This call borrows the `UnsafeCell` mutably (at compile-time) which
1337	/// guarantees that we possess the only reference.
1338	///
1339	/// This is like [`UnsafeCell::get_mut`], but for `Ptr`.
1340	///
1341	/// [`UnsafeCell::get_mut`]: core::cell::UnsafeCell::get_mut
1342	#[must_use]
1343	#[inline(always)]
1344	pub fn get_mut(self) -> Ptr<'a, T, I> {
1345	// SAFETY:
1346	// - The closure uses an `as` cast, which preserves address range
1347	// and provenance.
1348	// - We require `I: Invariants<Aliasing = Exclusive>`, so we are not
1349	// required to uphold `UnsafeCell` equality.
1350	#[allow(clippy::as_conversions)]
1351	let ptr = unsafe { self.cast_unsized(\|p\| p as *mut T) };
1352
1353	// SAFETY: `UnsafeCell<T>` has the same alignment as `T` [1],
1354	// and so if `self` is guaranteed to be aligned, then so is the
1355	// returned `Ptr`.
1356	//
1357	// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout:
1358	//
1359	// `UnsafeCell<T>` has the same in-memory representation as
1360	// its inner type `T`. A consequence of this guarantee is that
1361	// it is possible to convert between `T` and `UnsafeCell<T>`.
1362	let ptr = unsafe { ptr.assume_alignment::<I::Alignment>() };
1363
1364	// SAFETY: `UnsafeCell<T>` has the same bit validity as `T` [1], and
1365	// so if `self` has a particular validity invariant, then the same
1366	// holds of the returned `Ptr`. Technically the term
1367	// "representation" doesn't guarantee this, but the subsequent
1368	// sentence in the documentation makes it clear that this is the
1369	// intention.
1370	//
1371	// [1] Per https://doc.rust-lang.org/1.81.0/core/cell/struct.UnsafeCell.html#memory-layout:
1372	//
1373	// `UnsafeCell<T>` has the same in-memory representation as its
1374	// inner type `T`. A consequence of this guarantee is that it is
1375	// possible to convert between `T` and `UnsafeCell<T>`.
1376	let ptr = unsafe { ptr.assume_validity::<I::Validity>() };
1377	ptr.unify_invariants()
1378	}
1379	}
1380	}
1381
1382	/// Projections through the referent.
1383	mod _project {
1384	use core::ops::Range;
1385
1386	#[allow(unused_imports)]
1387	use crate::util::polyfills::NumExt as _;
1388
1389	use super::*;
1390
1391	impl<'a, T, I> Ptr<'a, T, I>
1392	where
1393	T: 'a + ?Sized,
1394	I: Invariants<Validity = Initialized>,
1395	{
1396	/// Projects a field from `self`.
1397	///
1398	/// # Safety
1399	///
1400	/// `project` has the same safety preconditions as `cast_unsized`.
1401	#[doc(hidden)]
1402	#[inline]
1403	pub unsafe fn project<U: 'a + ?Sized>(
1404	self,
1405	projector: impl FnOnce(*mut T) -> *mut U,
1406	) -> Ptr<'a, U, (I::Aliasing, Any, Initialized)> {
1407	// TODO(#1122): If `cast_unsized` were able to reason that, when
1408	// casting from an `Initialized` pointer, the result is another
1409	// `Initialized` pointer, we could remove this method entirely.
1410
1411	// SAFETY: This method has the same safety preconditions as
1412	// `cast_unsized`.
1413	let ptr = unsafe { self.cast_unsized(projector) };
1414
1415	// SAFETY: If all of the bytes of `self` are initialized (as
1416	// promised by `I: Invariants<Validity = Initialized>`), then any
1417	// subset of those bytes are also all initialized.
1418	unsafe { ptr.assume_validity::<Initialized>() }
1419	}
1420	}
1421
1422	impl<'a, T, I> Ptr<'a, T, I>
1423	where
1424	T: 'a + KnownLayout<PointerMetadata = usize> + ?Sized,
1425	I: Invariants,
1426	{
1427	/// The number of trailing slice elements in the object referenced by
1428	/// `self`.
1429	///
1430	/// # Safety
1431	///
1432	/// Unsafe code my rely on `trailing_slice_len` satisfying the above
1433	/// contract.
1434	pub(super) fn trailing_slice_len(&self) -> usize {
1435	T::pointer_to_metadata(self.as_non_null().as_ptr())
1436	}
1437	}
1438
1439	impl<'a, T, I> Ptr<'a, [T], I>
1440	where
1441	T: 'a,
1442	I: Invariants,
1443	{
1444	/// The number of slice elements in the object referenced by `self`.
1445	///
1446	/// # Safety
1447	///
1448	/// Unsafe code my rely on `len` satisfying the above contract.
1449	pub(crate) fn len(&self) -> usize {
1450	self.trailing_slice_len()
1451	}
1452
1453	/// Creates a pointer which addresses the given `range` of self.
1454	///
1455	/// # Safety
1456	///
1457	/// `range` is a valid range (`start <= end`) and `end <= self.len()`.
1458	pub(crate) unsafe fn slice_unchecked(self, range: Range<usize>) -> Self {
1459	let base = self.as_non_null().cast::<T>().as_ptr();
1460
1461	// SAFETY: The caller promises that `start <= end <= self.len()`. By
1462	// invariant, if `self`'s referent is not zero-sized, then `self`
1463	// refers to a byte range which is contained within a single
1464	// allocation, which is no more than `isize::MAX` bytes long, and
1465	// which does not wrap around the address space. Thus, this pointer
1466	// arithmetic remains in-bounds of the same allocation, and does not
1467	// wrap around the address space. The offset (in bytes) does not
1468	// overflow `isize`.
1469	//
1470	// If `self`'s referent is zero-sized, then these conditions are
1471	// trivially satisfied.
1472	let base = unsafe { base.add(range.start) };
1473
1474	// SAFETY: The caller promises that `start <= end`, and so this will
1475	// not underflow.
1476	#[allow(unstable_name_collisions, clippy::incompatible_msrv)]
1477	let len = unsafe { range.end.unchecked_sub(range.start) };
1478
1479	let ptr = core::ptr::slice_from_raw_parts_mut(base, len);
1480
1481	// SAFETY: By invariant, `self`'s address is non-null and its range
1482	// does not wrap around the address space. Since, by the preceding
1483	// lemma, `ptr` addresses a range within that addressed by `self`,
1484	// `ptr` is non-null.
1485	let ptr = unsafe { NonNull::new_unchecked(ptr) };
1486
1487	// SAFETY:
1488	//
1489	// Lemma 0: `ptr` addresses a subset of the bytes addressed by
1490	// `self`, and has the same provenance.
1491	// Proof: The caller guarantees that `start <= end <= self.len()`.
1492	// Thus, `base` is in-bounds of `self`, and `base + (end -
1493	// start)` is also in-bounds of self. Finally, `ptr` is
1494	// constructed using provenance-preserving operations.
1495	//
1496	// 0. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is
1497	// not zero sized, then `ptr` is derived from some valid Rust
1498	// allocation, `A`.
1499	// 1. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is
1500	// not zero sized, then `ptr` has valid provenance for `A`.
1501	// 2. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is
1502	// not zero sized, then `ptr` addresses a byte range which is
1503	// entirely contained in `A`.
1504	// 3. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte
1505	// range whose length fits in an `isize`.
1506	// 4. Per Lemma 0 and by invariant on `self`, `ptr` addresses a byte
1507	// range which does not wrap around the address space.
1508	// 5. Per Lemma 0 and by invariant on `self`, if `ptr`'s referent is
1509	// not zero sized, then `A` is guaranteed to live for at least
1510	// `'a`.
1511	// 6. Per Lemma 0 and by invariant on `self`, `ptr` conforms to the
1512	// aliasing invariant of [`I::Aliasing`](invariant::Aliasing).
1513	// 7. Per Lemma 0 and by invariant on `self`, `ptr` conforms to the
1514	// alignment invariant of [`I::Alignment`](invariant::Alignment).
1515	// 8. Per Lemma 0 and by invariant on `self`, `ptr` conforms to the
1516	// validity invariant of [`I::Validity`](invariant::Validity).
1517	unsafe { Ptr::new(ptr) }
1518	}
1519
1520	/// Splits the slice in two.
1521	///
1522	/// # Safety
1523	///
1524	/// The caller promises that `l_len <= self.len()`.
1525	pub(crate) unsafe fn split_at(self, l_len: usize) -> (Self, Self) {
1526	// SAFETY: `Any` imposes no invariants, and so this is always sound.
1527	let slf = unsafe { self.assume_aliasing::<Any>() };
1528
1529	// SAFETY: The caller promises that `l_len <= self.len()`.
1530	// Trivially, `0 <= l_len`.
1531	let left = unsafe { slf.slice_unchecked(`0`..l_len) };
1532
1533	// SAFETY: The caller promises that `l_len <= self.len() =
1534	// slf.len()`. Trivially, `slf.len() <= slf.len()`.
1535	let right = unsafe { slf.slice_unchecked(l_len..slf.len()) };
1536
1537	// LEMMA: `left` and `right` are non-overlapping. Proof: `left` is
1538	// constructed from `slf` with `l_len` as its (exclusive) upper
1539	// bound, while `right` is constructed from `slf` with `l_len` as
1540	// its (inclusive) lower bound. Thus, no index is a member of both
1541	// ranges.
1542
1543	// SAFETY: By the preceding lemma, `left` and `right` do not alias.
1544	// We do not construct any other `Ptr`s or references which alias
1545	// `left` or `right`. Thus, the only `Ptr`s or references which
1546	// alias `left` or `right` are outside of this method. By invariant,
1547	// `self` obeys the aliasing invariant `I::Aliasing` with respect to
1548	// those other `Ptr`s or references, and so `left` and `right` do as
1549	// well.
1550	let (left, right) = unsafe {
1551	(left.assume_aliasing::<I::Aliasing>(), right.assume_aliasing::<I::Aliasing>())
1552	};
1553	(left.unify_invariants(), right.unify_invariants())
1554	}
1555
1556	/// Iteratively projects the elements `Ptr<T>` from `Ptr<[T]>`.
1557	pub(crate) fn iter(&self) -> impl Iterator<Item = Ptr<'a, T, I>> {
1558	// TODO(#429): Once `NonNull::cast` documents that it preserves
1559	// provenance, cite those docs.
1560	let base = self.as_non_null().cast::<T>().as_ptr();
1561	(`0`..self.len()).map(move \|i\| {
1562	// TODO(https://github.com/rust-lang/rust/issues/74265): Use
1563	// `NonNull::get_unchecked_mut`.
1564
1565	// SAFETY: If the following conditions are not satisfied
1566	// `pointer::cast` may induce Undefined Behavior [1]:
1567	//
1568	// > - The computed offset, `count size_of::<T>()` bytes, must*
1569	// > not overflow `isize``.
1570	// > - If the computed offset is non-zero, then `self` must be
1571	// > derived from a pointer to some allocated object, and the
1572	// > entire memory range between `self` and the result must be
1573	// > in bounds of that allocated object. In particular, this
1574	// > range must not “wrap around” the edge of the address
1575	// > space.
1576	//
1577	// [1] https://doc.rust-lang.org/std/primitive.pointer.html#method.add
1578	//
1579	// We satisfy both of these conditions here:
1580	// - By invariant on `Ptr`, `self` addresses a byte range whose
1581	// length fits in an `isize`. Since `elem` is contained in
1582	// `self`, the computed offset of `elem` must fit within
1583	// `isize.`
1584	// - If the computed offset is non-zero, then this means that
1585	// the referent is not zero-sized. In this case, `base` points
1586	// to an allocated object (by invariant on `self`). Thus:
1587	// - By contract, `self.len()` accurately reflects the number
1588	// of elements in the slice. `i` is in bounds of `c.len()`
1589	// by construction, and so the result of this addition
1590	// cannot overflow past the end of the allocation referred
1591	// to by `c`.
1592	// - By invariant on `Ptr`, `self` addresses a byte range
1593	// which does not wrap around the address space. Since
1594	// `elem` is contained in `self`, the computed offset of
1595	// `elem` must wrap around the address space.
1596	//
1597	// TODO(#429): Once `pointer::add` documents that it preserves
1598	// provenance, cite those docs.
1599	let elem = unsafe { base.add(i) };
1600
1601	// SAFETY:
1602	// - `elem` must not be null. `base` is constructed from a
1603	// `NonNull` pointer, and the addition that produces `elem`
1604	// must not overflow or wrap around, so `elem >= base > 0`.
1605	//
1606	// TODO(#429): Once `NonNull::new_unchecked` documents that it
1607	// preserves provenance, cite those docs.
1608	let elem = unsafe { NonNull::new_unchecked(elem) };
1609
1610	// SAFETY: The safety invariants of `Ptr::new` (see definition)
1611	// are satisfied:
1612	// 0. If `elem`'s referent is not zero sized, then `elem` is
1613	// derived from a valid Rust allocation, because `self` is
1614	// derived from a valid Rust allocation, by invariant on
1615	// `Ptr`.
1616	// 1. If `elem`'s referent is not zero sized, then `elem` has
1617	// valid provenance for `self`, because it derived from
1618	// `self` using a series of provenance-preserving operations.
1619	// 2. If `elem`'s referent is not zero sized, then `elem` is
1620	// entirely contained in the allocation of `self` (see
1621	// above).
1622	// 3. `elem` addresses a byte range whose length fits in an
1623	// `isize` (see above).
1624	// 4. `elem` addresses a byte range which does not wrap around
1625	// the address space (see above).
1626	// 5. If `elem`'s referent is not zero sized, then the
1627	// allocation of `elem` is guaranteed to live for at least
1628	// `'a`, because `elem` is entirely contained in `self`,
1629	// which lives for at least `'a` by invariant on `Ptr`.
1630	// 6. `elem` conforms to the aliasing invariant of `I::Aliasing`
1631	// because projection does not impact the aliasing invariant.
1632	// 7. `elem`, conditionally, conforms to the validity invariant
1633	// of `I::Alignment`. If `elem` is projected from data
1634	// well-aligned for `[T]`, `elem` will be valid for `T`.
1635	// 8. `elem`, conditionally, conforms to the validity invariant
1636	// of `I::Validity`. If `elem` is projected from data valid
1637	// for `[T]`, `elem` will be valid for `T`.
1638	unsafe { Ptr::new(elem) }
1639	})
1640	}
1641	}
1642	}
1643
1644	#[cfg(test)]
1645	mod tests {
1646	use core::mem::{self, MaybeUninit};
1647
1648	use static_assertions::{assert_impl_all, assert_not_impl_any};
1649
1650	use super::*;
1651	use crate::{pointer::BecauseImmutable, util::testutil::AU64, FromBytes, Immutable};
1652
1653	#[test]
1654	fn test_split_at() {
1655	const N: usize = `16`;
1656	let mut arr = [`1`; N];
1657	let mut ptr = Ptr::from_mut(&mut arr).as_slice();
1658	for i in `0`..=N {
1659	assert_eq!(ptr.len(), N);
1660	// SAFETY: `i` is in bounds by construction.
1661	let (l, r) = unsafe { ptr.reborrow().split_at(i) };
1662	let l_sum: usize = l.iter().map(Ptr::read_unaligned::<BecauseImmutable>).sum();
1663	let r_sum: usize = r.iter().map(Ptr::read_unaligned::<BecauseImmutable>).sum();
1664	assert_eq!(l_sum, i);
1665	assert_eq!(r_sum, N - i);
1666	assert_eq!(l_sum + r_sum, N);
1667	}
1668	}
1669
1670	mod test_ptr_try_cast_into_soundness {
1671	use super::*;
1672
1673	// This test is designed so that if `Ptr::try_cast_into_xxx` are
1674	// buggy, it will manifest as unsoundness that Miri can detect.
1675
1676	// - If `size_of::<T>() == 0`, `N == 4`
1677	// - Else, `N == 4 size_of::<T>()`*
1678	//
1679	// Each test will be run for each metadata in `metas`.
1680	fn test<T, I, const N: usize>(metas: I)
1681	where
1682	T: ?Sized + KnownLayout + Immutable + FromBytes,
1683	I: IntoIterator<Item = Option<T::PointerMetadata>> + Clone,
1684	{
1685	let mut bytes = [MaybeUninit::<u8>::uninit(); N];
1686	let initialized = [MaybeUninit::new(`0u8`); N];
1687	for start in `0`..=bytes.len() {
1688	for end in start..=bytes.len() {
1689	// Set all bytes to uninitialized other than those in
1690	// the range we're going to pass to `try_cast_from`.
1691	// This allows Miri to detect out-of-bounds reads
1692	// because they read uninitialized memory. Without this,
1693	// some out-of-bounds reads would still be in-bounds of
1694	// `bytes`, and so might spuriously be accepted.
1695	bytes = [MaybeUninit::<u8>::uninit(); N];
1696	let bytes = &mut bytes[start..end];
1697	// Initialize only the byte range we're going to pass to
1698	// `try_cast_from`.
1699	bytes.copy_from_slice(&initialized[start..end]);
1700
1701	let bytes = {
1702	let bytes: *const [MaybeUninit<u8>] = bytes;
1703	#[allow(clippy::as_conversions)]
1704	let bytes = bytes as *const [u8];
1705	// SAFETY: We just initialized these bytes to valid
1706	// `u8`s.
1707	unsafe { &*bytes }
1708	};
1709
1710	// SAFETY: The bytes in `slf` must be initialized.
1711	unsafe fn validate_and_get_len<T: ?Sized + KnownLayout + FromBytes>(
1712	slf: Ptr<'_, T, (Shared, Aligned, Initialized)>,
1713	) -> usize {
1714	let t = slf.bikeshed_recall_valid().as_ref();
1715
1716	let bytes = {
1717	let len = mem::size_of_val(t);
1718	let t: *const T = t;
1719	// SAFETY:
1720	// - We know `t`'s bytes are all initialized
1721	// because we just read it from `slf`, which
1722	// points to an initialized range of bytes. If
1723	// there's a bug and this doesn't hold, then
1724	// that's exactly what we're hoping Miri will
1725	// catch!
1726	// - Since `T: FromBytes`, `T` doesn't contain
1727	// any `UnsafeCell`s, so it's okay for `t: T`
1728	// and a `&[u8]` to the same memory to be
1729	// alive concurrently.
1730	unsafe { core::slice::from_raw_parts(t.cast::<u8>(), len) }
1731	};
1732
1733	// This assertion ensures that `t`'s bytes are read
1734	// and compared to another value, which in turn
1735	// ensures that Miri gets a chance to notice if any
1736	// of `t`'s bytes are uninitialized, which they
1737	// shouldn't be (see the comment above).
1738	assert_eq!(bytes, vec![`0u8`; bytes.len()]);
1739
1740	mem::size_of_val(t)
1741	}
1742
1743	for meta in metas.clone().into_iter() {
1744	for cast_type in [CastType::Prefix, CastType::Suffix] {
1745	if let Ok((slf, remaining)) = Ptr::from_ref(bytes)
1746	.try_cast_into::<T, BecauseImmutable>(cast_type, meta)
1747	{
1748	// SAFETY: All bytes in `bytes` have been
1749	// initialized.
1750	let len = unsafe { validate_and_get_len(slf) };
1751	assert_eq!(remaining.len(), bytes.len() - len);
1752	#[allow(unstable_name_collisions)]
1753	let bytes_addr = bytes.as_ptr().addr();
1754	#[allow(unstable_name_collisions)]
1755	let remaining_addr = remaining.as_non_null().as_ptr().addr();
1756	match cast_type {
1757	CastType::Prefix => {
1758	assert_eq!(remaining_addr, bytes_addr + len)
1759	}
1760	CastType::Suffix => assert_eq!(remaining_addr, bytes_addr),
1761	}
1762
1763	if let Some(want) = meta {
1764	let got = KnownLayout::pointer_to_metadata(
1765	slf.as_non_null().as_ptr(),
1766	);
1767	assert_eq!(got, want);
1768	}
1769	}
1770	}
1771
1772	if let Ok(slf) = Ptr::from_ref(bytes)
1773	.try_cast_into_no_leftover::<T, BecauseImmutable>(meta)
1774	{
1775	// SAFETY: All bytes in `bytes` have been
1776	// initialized.
1777	let len = unsafe { validate_and_get_len(slf) };
1778	assert_eq!(len, bytes.len());
1779
1780	if let Some(want) = meta {
1781	let got =
1782	KnownLayout::pointer_to_metadata(slf.as_non_null().as_ptr());
1783	assert_eq!(got, want);
1784	}
1785	}
1786	}
1787	}
1788	}
1789	}
1790
1791	#[derive(FromBytes, KnownLayout, Immutable)]
1792	#[repr(C)]
1793	struct SliceDst<T> {
1794	a: u8,
1795	trailing: [T],
1796	}
1797
1798	// Each test case becomes its own `#[test]` function. We do this because
1799	// this test in particular takes far, far longer to execute under Miri
1800	// than all of our other tests combined. Previously, we had these
1801	// execute sequentially in a single test function. We run Miri tests in
1802	// parallel in CI, but this test being sequential meant that most of
1803	// that parallelism was wasted, as all other tests would finish in a
1804	// fraction of the total execution time, leaving this test to execute on
1805	// a single thread for the remainder of the test. By putting each test
1806	// case in its own function, we permit better use of available
1807	// parallelism.
1808	macro_rules! test {
1809	($test_name:ident: $ty:ty) => {
1810	#[test]
1811	#[allow(non_snake_case)]
1812	fn $test_name() {
1813	const S: usize = core::mem::size_of::<$ty>();
1814	const N: usize = if S == `0` { `4` } else { S * `4` };
1815	test::<$ty, _, N>([None]);
1816
1817	// If `$ty` is a ZST, then we can't pass `None` as the
1818	// pointer metadata, or else computing the correct trailing
1819	// slice length will panic.
1820	if S == `0` {
1821	test::<[$ty], _, N>([Some(`0`), Some(`1`), Some(`2`), Some(`3`)]);
1822	test::<SliceDst<$ty>, _, N>([Some(`0`), Some(`1`), Some(`2`), Some(`3`)]);
1823	} else {
1824	test::<[$ty], _, N>([None, Some(`0`), Some(`1`), Some(`2`), Some(`3`)]);
1825	test::<SliceDst<$ty>, _, N>([None, Some(`0`), Some(`1`), Some(`2`), Some(`3`)]);
1826	}
1827	}
1828	};
1829	($ty:ident) => {
1830	test!($ty: $ty);
1831	};
1832	($($ty:ident),) => { $(test!($ty);) }
1833	}
1834
1835	test!(empty_tuple: ());
1836	test!(u8, u16, u32, u64, u128, usize, AU64);
1837	test!(i8, i16, i32, i64, i128, isize);
1838	test!(f32, f64);
1839	}
1840
1841	#[test]
1842	fn test_invariants() {
1843	// Test that the correct invariant relationships hold.
1844	use super::invariant::*;
1845
1846	assert_not_impl_any!(Any: AtLeast<Shared>);
1847	assert_impl_all!(Shared: AtLeast<Shared>);
1848	assert_impl_all!(Exclusive: AtLeast<Shared>);
1849
1850	assert_not_impl_any!(Any: AtLeast<AsInitialized>);
1851	assert_impl_all!(AsInitialized: AtLeast<AsInitialized>);
1852	assert_impl_all!(Initialized: AtLeast<AsInitialized>);
1853	assert_impl_all!(Valid: AtLeast<AsInitialized>);
1854	}
1855
1856	#[test]
1857	fn test_try_cast_into_explicit_count() {
1858	macro_rules! test {
1859	($ty:ty, $bytes:expr, $elems:expr, $expect:expr) => {{
1860	let bytes = [`0u8`; $bytes];
1861	let ptr = Ptr::from_ref(&bytes[..]);
1862	let res =
1863	ptr.try_cast_into::<$ty, BecauseImmutable>(CastType::Prefix, Some($elems));
1864	if let Some(expect) = $expect {
1865	let (ptr, _) = res.unwrap();
1866	assert_eq!(
1867	KnownLayout::pointer_to_metadata(ptr.as_non_null().as_ptr()),
1868	expect
1869	);
1870	} else {
1871	let _ = res.unwrap_err();
1872	}
1873	}};
1874	}
1875
1876	#[derive(KnownLayout, Immutable)]
1877	#[repr(C)]
1878	struct ZstDst {
1879	u: [u8; `8`],
1880	slc: [()],
1881	}
1882
1883	test!(ZstDst, `8`, `0`, Some(`0`));
1884	test!(ZstDst, `7`, `0`, None);
1885
1886	test!(ZstDst, `8`, usize::MAX, Some(usize::MAX));
1887	test!(ZstDst, `7`, usize::MAX, None);
1888
1889	#[derive(KnownLayout, Immutable)]
1890	#[repr(C)]
1891	struct Dst {
1892	u: [u8; `8`],
1893	slc: [u8],
1894	}
1895
1896	test!(Dst, `8`, `0`, Some(`0`));
1897	test!(Dst, `7`, `0`, None);
1898
1899	test!(Dst, `9`, `1`, Some(`1`));
1900	test!(Dst, `8`, `1`, None);
1901
1902	// If we didn't properly check for overflow, this would cause the
1903	// metadata to overflow to 0, and thus the cast would spuriously
1904	// succeed.
1905	test!(Dst, `8`, usize::MAX - `8` + `1`, None);
1906	}
1907	}
1908