rwlock.rs - Codebrowser

1	// Copyright 2016 Amanieu d'Antras
2	//
3	// Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4	// http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5	// http://opensource.org/licenses/MIT>, at your option. This file may not be
6	// copied, modified, or distributed except according to those terms.
7
8	use core::cell::UnsafeCell;
9	use core::fmt;
10	use core::marker::PhantomData;
11	use core::mem;
12	use core::ops::{Deref, DerefMut};
13
14	#[cfg(feature = "arc_lock")]
15	use alloc::sync::Arc;
16	#[cfg(feature = "arc_lock")]
17	use core::mem::ManuallyDrop;
18	#[cfg(feature = "arc_lock")]
19	use core::ptr;
20
21	#[cfg(feature = "owning_ref")]
22	use owning_ref::StableAddress;
23
24	#[cfg(feature = "serde")]
25	use serde::{Deserialize, Deserializer, Serialize, Serializer};
26
27	/// Basic operations for a reader-writer lock.
28	///
29	/// Types implementing this trait can be used by `RwLock` to form a safe and
30	/// fully-functioning `RwLock` type.
31	///
32	/// # Safety
33	///
34	/// Implementations of this trait must ensure that the `RwLock` is actually
35	/// exclusive: an exclusive lock can't be acquired while an exclusive or shared
36	/// lock exists, and a shared lock can't be acquire while an exclusive lock
37	/// exists.
38	pub unsafe trait RawRwLock {
39	/// Initial value for an unlocked `RwLock`.
40	// A “non-constant” const item is a legacy way to supply an initialized value to downstream
41	// static items. Can hopefully be replaced with `const fn new() -> Self` at some point.
42	#[allow(clippy::declare_interior_mutable_const)]
43	const INIT: Self;
44
45	/// Marker type which determines whether a lock guard should be `Send`. Use
46	/// one of the `GuardSend` or `GuardNoSend` helper types here.
47	type GuardMarker;
48
49	/// Acquires a shared lock, blocking the current thread until it is able to do so.
50	fn lock_shared(&self);
51
52	/// Attempts to acquire a shared lock without blocking.
53	fn try_lock_shared(&self) -> bool;
54
55	/// Releases a shared lock.
56	///
57	/// # Safety
58	///
59	/// This method may only be called if a shared lock is held in the current context.
60	unsafe fn unlock_shared(&self);
61
62	/// Acquires an exclusive lock, blocking the current thread until it is able to do so.
63	fn lock_exclusive(&self);
64
65	/// Attempts to acquire an exclusive lock without blocking.
66	fn try_lock_exclusive(&self) -> bool;
67
68	/// Releases an exclusive lock.
69	///
70	/// # Safety
71	///
72	/// This method may only be called if an exclusive lock is held in the current context.
73	unsafe fn unlock_exclusive(&self);
74
75	/// Checks if this `RwLock` is currently locked in any way.
76	#[inline]
77	fn is_locked(&self) -> bool {
78	let acquired_lock = self.try_lock_exclusive();
79	if acquired_lock {
80	// Safety: A lock was successfully acquired above.
81	unsafe {
82	self.unlock_exclusive();
83	}
84	}
85	!acquired_lock
86	}
87
88	/// Check if this `RwLock` is currently exclusively locked.
89	fn is_locked_exclusive(&self) -> bool {
90	let acquired_lock = self.try_lock_shared();
91	if acquired_lock {
92	// Safety: A shared lock was successfully acquired above.
93	unsafe {
94	self.unlock_shared();
95	}
96	}
97	!acquired_lock
98	}
99	}
100
101	/// Additional methods for RwLocks which support fair unlocking.
102	///
103	/// Fair unlocking means that a lock is handed directly over to the next waiting
104	/// thread if there is one, without giving other threads the opportunity to
105	/// "steal" the lock in the meantime. This is typically slower than unfair
106	/// unlocking, but may be necessary in certain circumstances.
107	pub unsafe trait RawRwLockFair: RawRwLock {
108	/// Releases a shared lock using a fair unlock protocol.
109	///
110	/// # Safety
111	///
112	/// This method may only be called if a shared lock is held in the current context.
113	unsafe fn unlock_shared_fair(&self);
114
115	/// Releases an exclusive lock using a fair unlock protocol.
116	///
117	/// # Safety
118	///
119	/// This method may only be called if an exclusive lock is held in the current context.
120	unsafe fn unlock_exclusive_fair(&self);
121
122	/// Temporarily yields a shared lock to a waiting thread if there is one.
123	///
124	/// This method is functionally equivalent to calling `unlock_shared_fair` followed
125	/// by `lock_shared`, however it can be much more efficient in the case where there
126	/// are no waiting threads.
127	///
128	/// # Safety
129	///
130	/// This method may only be called if a shared lock is held in the current context.
131	unsafe fn bump_shared(&self) {
132	self.unlock_shared_fair();
133	self.lock_shared();
134	}
135
136	/// Temporarily yields an exclusive lock to a waiting thread if there is one.
137	///
138	/// This method is functionally equivalent to calling `unlock_exclusive_fair` followed
139	/// by `lock_exclusive`, however it can be much more efficient in the case where there
140	/// are no waiting threads.
141	///
142	/// # Safety
143	///
144	/// This method may only be called if an exclusive lock is held in the current context.
145	unsafe fn bump_exclusive(&self) {
146	self.unlock_exclusive_fair();
147	self.lock_exclusive();
148	}
149	}
150
151	/// Additional methods for RwLocks which support atomically downgrading an
152	/// exclusive lock to a shared lock.
153	pub unsafe trait RawRwLockDowngrade: RawRwLock {
154	/// Atomically downgrades an exclusive lock into a shared lock without
155	/// allowing any thread to take an exclusive lock in the meantime.
156	///
157	/// # Safety
158	///
159	/// This method may only be called if an exclusive lock is held in the current context.
160	unsafe fn downgrade(&self);
161	}
162
163	/// Additional methods for RwLocks which support locking with timeouts.
164	///
165	/// The `Duration` and `Instant` types are specified as associated types so that
166	/// this trait is usable even in `no_std` environments.
167	pub unsafe trait RawRwLockTimed: RawRwLock {
168	/// Duration type used for `try_lock_for`.
169	type Duration;
170
171	/// Instant type used for `try_lock_until`.
172	type Instant;
173
174	/// Attempts to acquire a shared lock until a timeout is reached.
175	fn try_lock_shared_for(&self, timeout: Self::Duration) -> bool;
176
177	/// Attempts to acquire a shared lock until a timeout is reached.
178	fn try_lock_shared_until(&self, timeout: Self::Instant) -> bool;
179
180	/// Attempts to acquire an exclusive lock until a timeout is reached.
181	fn try_lock_exclusive_for(&self, timeout: Self::Duration) -> bool;
182
183	/// Attempts to acquire an exclusive lock until a timeout is reached.
184	fn try_lock_exclusive_until(&self, timeout: Self::Instant) -> bool;
185	}
186
187	/// Additional methods for RwLocks which support recursive read locks.
188	///
189	/// These are guaranteed to succeed without blocking if
190	/// another read lock is held at the time of the call. This allows a thread
191	/// to recursively lock a `RwLock`. However using this method can cause
192	/// writers to starve since readers no longer block if a writer is waiting
193	/// for the lock.
194	pub unsafe trait RawRwLockRecursive: RawRwLock {
195	/// Acquires a shared lock without deadlocking in case of a recursive lock.
196	fn lock_shared_recursive(&self);
197
198	/// Attempts to acquire a shared lock without deadlocking in case of a recursive lock.
199	fn try_lock_shared_recursive(&self) -> bool;
200	}
201
202	/// Additional methods for RwLocks which support recursive read locks and timeouts.
203	pub unsafe trait RawRwLockRecursiveTimed: RawRwLockRecursive + RawRwLockTimed {
204	/// Attempts to acquire a shared lock until a timeout is reached, without
205	/// deadlocking in case of a recursive lock.
206	fn try_lock_shared_recursive_for(&self, timeout: Self::Duration) -> bool;
207
208	/// Attempts to acquire a shared lock until a timeout is reached, without
209	/// deadlocking in case of a recursive lock.
210	fn try_lock_shared_recursive_until(&self, timeout: Self::Instant) -> bool;
211	}
212
213	/// Additional methods for RwLocks which support atomically upgrading a shared
214	/// lock to an exclusive lock.
215	///
216	/// This requires acquiring a special "upgradable read lock" instead of a
217	/// normal shared lock. There may only be one upgradable lock at any time,
218	/// otherwise deadlocks could occur when upgrading.
219	pub unsafe trait RawRwLockUpgrade: RawRwLock {
220	/// Acquires an upgradable lock, blocking the current thread until it is able to do so.
221	fn lock_upgradable(&self);
222
223	/// Attempts to acquire an upgradable lock without blocking.
224	fn try_lock_upgradable(&self) -> bool;
225
226	/// Releases an upgradable lock.
227	///
228	/// # Safety
229	///
230	/// This method may only be called if an upgradable lock is held in the current context.
231	unsafe fn unlock_upgradable(&self);
232
233	/// Upgrades an upgradable lock to an exclusive lock.
234	///
235	/// # Safety
236	///
237	/// This method may only be called if an upgradable lock is held in the current context.
238	unsafe fn upgrade(&self);
239
240	/// Attempts to upgrade an upgradable lock to an exclusive lock without
241	/// blocking.
242	///
243	/// # Safety
244	///
245	/// This method may only be called if an upgradable lock is held in the current context.
246	unsafe fn try_upgrade(&self) -> bool;
247	}
248
249	/// Additional methods for RwLocks which support upgradable locks and fair
250	/// unlocking.
251	pub unsafe trait RawRwLockUpgradeFair: RawRwLockUpgrade + RawRwLockFair {
252	/// Releases an upgradable lock using a fair unlock protocol.
253	///
254	/// # Safety
255	///
256	/// This method may only be called if an upgradable lock is held in the current context.
257	unsafe fn unlock_upgradable_fair(&self);
258
259	/// Temporarily yields an upgradable lock to a waiting thread if there is one.
260	///
261	/// This method is functionally equivalent to calling `unlock_upgradable_fair` followed
262	/// by `lock_upgradable`, however it can be much more efficient in the case where there
263	/// are no waiting threads.
264	///
265	/// # Safety
266	///
267	/// This method may only be called if an upgradable lock is held in the current context.
268	unsafe fn bump_upgradable(&self) {
269	self.unlock_upgradable_fair();
270	self.lock_upgradable();
271	}
272	}
273
274	/// Additional methods for RwLocks which support upgradable locks and lock
275	/// downgrading.
276	pub unsafe trait RawRwLockUpgradeDowngrade: RawRwLockUpgrade + RawRwLockDowngrade {
277	/// Downgrades an upgradable lock to a shared lock.
278	///
279	/// # Safety
280	///
281	/// This method may only be called if an upgradable lock is held in the current context.
282	unsafe fn downgrade_upgradable(&self);
283
284	/// Downgrades an exclusive lock to an upgradable lock.
285	///
286	/// # Safety
287	///
288	/// This method may only be called if an exclusive lock is held in the current context.
289	unsafe fn downgrade_to_upgradable(&self);
290	}
291
292	/// Additional methods for RwLocks which support upgradable locks and locking
293	/// with timeouts.
294	pub unsafe trait RawRwLockUpgradeTimed: RawRwLockUpgrade + RawRwLockTimed {
295	/// Attempts to acquire an upgradable lock until a timeout is reached.
296	fn try_lock_upgradable_for(&self, timeout: Self::Duration) -> bool;
297
298	/// Attempts to acquire an upgradable lock until a timeout is reached.
299	fn try_lock_upgradable_until(&self, timeout: Self::Instant) -> bool;
300
301	/// Attempts to upgrade an upgradable lock to an exclusive lock until a
302	/// timeout is reached.
303	///
304	/// # Safety
305	///
306	/// This method may only be called if an upgradable lock is held in the current context.
307	unsafe fn try_upgrade_for(&self, timeout: Self::Duration) -> bool;
308
309	/// Attempts to upgrade an upgradable lock to an exclusive lock until a
310	/// timeout is reached.
311	///
312	/// # Safety
313	///
314	/// This method may only be called if an upgradable lock is held in the current context.
315	unsafe fn try_upgrade_until(&self, timeout: Self::Instant) -> bool;
316	}
317
318	/// A reader-writer lock
319	///
320	/// This type of lock allows a number of readers or at most one writer at any
321	/// point in time. The write portion of this lock typically allows modification
322	/// of the underlying data (exclusive access) and the read portion of this lock
323	/// typically allows for read-only access (shared access).
324	///
325	/// The type parameter `T` represents the data that this lock protects. It is
326	/// required that `T` satisfies `Send` to be shared across threads and `Sync` to
327	/// allow concurrent access through readers. The RAII guards returned from the
328	/// locking methods implement `Deref` (and `DerefMut` for the `write` methods)
329	/// to allow access to the contained of the lock.
330	pub struct RwLock<R, T: ?Sized> {
331	raw: R,
332	data: UnsafeCell<T>,
333	}
334
335	// Copied and modified from serde
336	#[cfg(feature = "serde")]
337	impl<R, T> Serialize for RwLock<R, T>
338	where
339	R: RawRwLock,
340	T: Serialize + ?Sized,
341	{
342	fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
343	where
344	S: Serializer,
345	{
346	self.read().serialize(serializer)
347	}
348	}
349
350	#[cfg(feature = "serde")]
351	impl<'de, R, T> Deserialize<'de> for RwLock<R, T>
352	where
353	R: RawRwLock,
354	T: Deserialize<'de> + ?Sized,
355	{
356	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
357	where
358	D: Deserializer<'de>,
359	{
360	Deserialize::deserialize(deserializer).map(RwLock::new)
361	}
362	}
363
364	unsafe impl<R: RawRwLock + Send, T: ?Sized + Send> Send for RwLock<R, T> {}
365	unsafe impl<R: RawRwLock + Sync, T: ?Sized + Send + Sync> Sync for RwLock<R, T> {}
366
367	impl<R: RawRwLock, T> RwLock<R, T> {
368	/// Creates a new instance of an `RwLock<T>` which is unlocked.
369	#[cfg(has_const_fn_trait_bound)]
370	#[inline]
371	pub const fn new(val: T) -> RwLock<R, T> {
372	RwLock {
373	data: UnsafeCell::new(val),
374	raw: R::INIT,
375	}
376	}
377
378	/// Creates a new instance of an `RwLock<T>` which is unlocked.
379	#[cfg(not(has_const_fn_trait_bound))]
380	#[inline]
381	pub fn new(val: T) -> RwLock<R, T> {
382	RwLock {
383	data: UnsafeCell::new(val),
384	raw: R::INIT,
385	}
386	}
387
388	/// Consumes this `RwLock`, returning the underlying data.
389	#[inline]
390	#[allow(unused_unsafe)]
391	pub fn into_inner(self) -> T {
392	unsafe { self.data.into_inner() }
393	}
394	}
395
396	impl<R, T> RwLock<R, T> {
397	/// Creates a new new instance of an `RwLock<T>` based on a pre-existing
398	/// `RawRwLock<T>`.
399	///
400	/// This allows creating a `RwLock<T>` in a constant context on stable
401	/// Rust.
402	#[inline]
403	pub const fn const_new(raw_rwlock: R, val: T) -> RwLock<R, T> {
404	RwLock {
405	data: UnsafeCell::new(val),
406	raw: raw_rwlock,
407	}
408	}
409	}
410
411	impl<R: RawRwLock, T: ?Sized> RwLock<R, T> {
412	/// Creates a new `RwLockReadGuard` without checking if the lock is held.
413	///
414	/// # Safety
415	///
416	/// This method must only be called if the thread logically holds a read lock.
417	///
418	/// This function does not increment the read count of the lock. Calling this function when a
419	/// guard has already been produced is undefined behaviour unless the guard was forgotten
420	/// with `mem::forget`.`
421	#[inline]
422	pub unsafe fn make_read_guard_unchecked(&self) -> RwLockReadGuard<'_, R, T> {
423	RwLockReadGuard {
424	rwlock: self,
425	marker: PhantomData,
426	}
427	}
428
429	/// Creates a new `RwLockReadGuard` without checking if the lock is held.
430	///
431	/// # Safety
432	///
433	/// This method must only be called if the thread logically holds a write lock.
434	///
435	/// Calling this function when a guard has already been produced is undefined behaviour unless
436	/// the guard was forgotten with `mem::forget`.
437	#[inline]
438	pub unsafe fn make_write_guard_unchecked(&self) -> RwLockWriteGuard<'_, R, T> {
439	RwLockWriteGuard {
440	rwlock: self,
441	marker: PhantomData,
442	}
443	}
444
445	/// Locks this `RwLock` with shared read access, blocking the current thread
446	/// until it can be acquired.
447	///
448	/// The calling thread will be blocked until there are no more writers which
449	/// hold the lock. There may be other readers currently inside the lock when
450	/// this method returns.
451	///
452	/// Note that attempts to recursively acquire a read lock on a `RwLock` when
453	/// the current thread already holds one may result in a deadlock.
454	///
455	/// Returns an RAII guard which will release this thread's shared access
456	/// once it is dropped.
457	#[inline]
458	pub fn read(&self) -> RwLockReadGuard<'_, R, T> {
459	self.raw.lock_shared();
460	// SAFETY: The lock is held, as required.
461	unsafe { self.make_read_guard_unchecked() }
462	}
463
464	/// Attempts to acquire this `RwLock` with shared read access.
465	///
466	/// If the access could not be granted at this time, then `None` is returned.
467	/// Otherwise, an RAII guard is returned which will release the shared access
468	/// when it is dropped.
469	///
470	/// This function does not block.
471	#[inline]
472	pub fn try_read(&self) -> Option<RwLockReadGuard<'_, R, T>> {
473	if self.raw.try_lock_shared() {
474	// SAFETY: The lock is held, as required.
475	Some(unsafe { self.make_read_guard_unchecked() })
476	} else {
477	None
478	}
479	}
480
481	/// Locks this `RwLock` with exclusive write access, blocking the current
482	/// thread until it can be acquired.
483	///
484	/// This function will not return while other writers or other readers
485	/// currently have access to the lock.
486	///
487	/// Returns an RAII guard which will drop the write access of this `RwLock`
488	/// when dropped.
489	#[inline]
490	pub fn write(&self) -> RwLockWriteGuard<'_, R, T> {
491	self.raw.lock_exclusive();
492	// SAFETY: The lock is held, as required.
493	unsafe { self.make_write_guard_unchecked() }
494	}
495
496	/// Attempts to lock this `RwLock` with exclusive write access.
497	///
498	/// If the lock could not be acquired at this time, then `None` is returned.
499	/// Otherwise, an RAII guard is returned which will release the lock when
500	/// it is dropped.
501	///
502	/// This function does not block.
503	#[inline]
504	pub fn try_write(&self) -> Option<RwLockWriteGuard<'_, R, T>> {
505	if self.raw.try_lock_exclusive() {
506	// SAFETY: The lock is held, as required.
507	Some(unsafe { self.make_write_guard_unchecked() })
508	} else {
509	None
510	}
511	}
512
513	/// Returns a mutable reference to the underlying data.
514	///
515	/// Since this call borrows the `RwLock` mutably, no actual locking needs to
516	/// take place---the mutable borrow statically guarantees no locks exist.
517	#[inline]
518	pub fn get_mut(&mut self) -> &mut T {
519	unsafe { &mut *self.data.get() }
520	}
521
522	/// Checks whether this `RwLock` is currently locked in any way.
523	#[inline]
524	pub fn is_locked(&self) -> bool {
525	self.raw.is_locked()
526	}
527
528	/// Check if this `RwLock` is currently exclusively locked.
529	#[inline]
530	pub fn is_locked_exclusive(&self) -> bool {
531	self.raw.is_locked_exclusive()
532	}
533
534	/// Forcibly unlocks a read lock.
535	///
536	/// This is useful when combined with `mem::forget` to hold a lock without
537	/// the need to maintain a `RwLockReadGuard` object alive, for example when
538	/// dealing with FFI.
539	///
540	/// # Safety
541	///
542	/// This method must only be called if the current thread logically owns a
543	/// `RwLockReadGuard` but that guard has be discarded using `mem::forget`.
544	/// Behavior is undefined if a rwlock is read-unlocked when not read-locked.
545	#[inline]
546	pub unsafe fn force_unlock_read(&self) {
547	self.raw.unlock_shared();
548	}
549
550	/// Forcibly unlocks a write lock.
551	///
552	/// This is useful when combined with `mem::forget` to hold a lock without
553	/// the need to maintain a `RwLockWriteGuard` object alive, for example when
554	/// dealing with FFI.
555	///
556	/// # Safety
557	///
558	/// This method must only be called if the current thread logically owns a
559	/// `RwLockWriteGuard` but that guard has be discarded using `mem::forget`.
560	/// Behavior is undefined if a rwlock is write-unlocked when not write-locked.
561	#[inline]
562	pub unsafe fn force_unlock_write(&self) {
563	self.raw.unlock_exclusive();
564	}
565
566	/// Returns the underlying raw reader-writer lock object.
567	///
568	/// Note that you will most likely need to import the `RawRwLock` trait from
569	/// `lock_api` to be able to call functions on the raw
570	/// reader-writer lock.
571	///
572	/// # Safety
573	///
574	/// This method is unsafe because it allows unlocking a mutex while
575	/// still holding a reference to a lock guard.
576	pub unsafe fn raw(&self) -> &R {
577	&self.raw
578	}
579
580	/// Returns a raw pointer to the underlying data.
581	///
582	/// This is useful when combined with `mem::forget` to hold a lock without
583	/// the need to maintain a `RwLockReadGuard` or `RwLockWriteGuard` object
584	/// alive, for example when dealing with FFI.
585	///
586	/// # Safety
587	///
588	/// You must ensure that there are no data races when dereferencing the
589	/// returned pointer, for example if the current thread logically owns a
590	/// `RwLockReadGuard` or `RwLockWriteGuard` but that guard has been discarded
591	/// using `mem::forget`.
592	#[inline]
593	pub fn data_ptr(&self) -> *mut T {
594	self.data.get()
595	}
596
597	/// Creates a new `RwLockReadGuard` without checking if the lock is held.
598	///
599	/// # Safety
600	///
601	/// This method must only be called if the thread logically holds a read lock.
602	///
603	/// This function does not increment the read count of the lock. Calling this function when a
604	/// guard has already been produced is undefined behaviour unless the guard was forgotten
605	/// with `mem::forget`.`
606	#[cfg(feature = "arc_lock")]
607	#[inline]
608	pub unsafe fn make_arc_read_guard_unchecked(self: &Arc<Self>) -> ArcRwLockReadGuard<R, T> {
609	ArcRwLockReadGuard {
610	rwlock: self.clone(),
611	marker: PhantomData,
612	}
613	}
614
615	/// Creates a new `RwLockWriteGuard` without checking if the lock is held.
616	///
617	/// # Safety
618	///
619	/// This method must only be called if the thread logically holds a write lock.
620	///
621	/// Calling this function when a guard has already been produced is undefined behaviour unless
622	/// the guard was forgotten with `mem::forget`.
623	#[cfg(feature = "arc_lock")]
624	#[inline]
625	pub unsafe fn make_arc_write_guard_unchecked(self: &Arc<Self>) -> ArcRwLockWriteGuard<R, T> {
626	ArcRwLockWriteGuard {
627	rwlock: self.clone(),
628	marker: PhantomData,
629	}
630	}
631
632	/// Locks this `RwLock` with read access, through an `Arc`.
633	///
634	/// This method is similar to the `read` method; however, it requires the `RwLock` to be inside of an `Arc`
635	/// and the resulting read guard has no lifetime requirements.
636	#[cfg(feature = "arc_lock")]
637	#[inline]
638	pub fn read_arc(self: &Arc<Self>) -> ArcRwLockReadGuard<R, T> {
639	self.raw.lock_shared();
640	// SAFETY: locking guarantee is upheld
641	unsafe { self.make_arc_read_guard_unchecked() }
642	}
643
644	/// Attempts to lock this `RwLock` with read access, through an `Arc`.
645	///
646	/// This method is similar to the `try_read` method; however, it requires the `RwLock` to be inside of an
647	/// `Arc` and the resulting read guard has no lifetime requirements.
648	#[cfg(feature = "arc_lock")]
649	#[inline]
650	pub fn try_read_arc(self: &Arc<Self>) -> Option<ArcRwLockReadGuard<R, T>> {
651	if self.raw.try_lock_shared() {
652	// SAFETY: locking guarantee is upheld
653	Some(unsafe { self.make_arc_read_guard_unchecked() })
654	} else {
655	None
656	}
657	}
658
659	/// Locks this `RwLock` with write access, through an `Arc`.
660	///
661	/// This method is similar to the `write` method; however, it requires the `RwLock` to be inside of an `Arc`
662	/// and the resulting write guard has no lifetime requirements.
663	#[cfg(feature = "arc_lock")]
664	#[inline]
665	pub fn write_arc(self: &Arc<Self>) -> ArcRwLockWriteGuard<R, T> {
666	self.raw.lock_exclusive();
667	// SAFETY: locking guarantee is upheld
668	unsafe { self.make_arc_write_guard_unchecked() }
669	}
670
671	/// Attempts to lock this `RwLock` with writ access, through an `Arc`.
672	///
673	/// This method is similar to the `try_write` method; however, it requires the `RwLock` to be inside of an
674	/// `Arc` and the resulting write guard has no lifetime requirements.
675	#[cfg(feature = "arc_lock")]
676	#[inline]
677	pub fn try_write_arc(self: &Arc<Self>) -> Option<ArcRwLockWriteGuard<R, T>> {
678	if self.raw.try_lock_exclusive() {
679	// SAFETY: locking guarantee is upheld
680	Some(unsafe { self.make_arc_write_guard_unchecked() })
681	} else {
682	None
683	}
684	}
685	}
686
687	impl<R: RawRwLockFair, T: ?Sized> RwLock<R, T> {
688	/// Forcibly unlocks a read lock using a fair unlock procotol.
689	///
690	/// This is useful when combined with `mem::forget` to hold a lock without
691	/// the need to maintain a `RwLockReadGuard` object alive, for example when
692	/// dealing with FFI.
693	///
694	/// # Safety
695	///
696	/// This method must only be called if the current thread logically owns a
697	/// `RwLockReadGuard` but that guard has be discarded using `mem::forget`.
698	/// Behavior is undefined if a rwlock is read-unlocked when not read-locked.
699	#[inline]
700	pub unsafe fn force_unlock_read_fair(&self) {
701	self.raw.unlock_shared_fair();
702	}
703
704	/// Forcibly unlocks a write lock using a fair unlock procotol.
705	///
706	/// This is useful when combined with `mem::forget` to hold a lock without
707	/// the need to maintain a `RwLockWriteGuard` object alive, for example when
708	/// dealing with FFI.
709	///
710	/// # Safety
711	///
712	/// This method must only be called if the current thread logically owns a
713	/// `RwLockWriteGuard` but that guard has be discarded using `mem::forget`.
714	/// Behavior is undefined if a rwlock is write-unlocked when not write-locked.
715	#[inline]
716	pub unsafe fn force_unlock_write_fair(&self) {
717	self.raw.unlock_exclusive_fair();
718	}
719	}
720
721	impl<R: RawRwLockTimed, T: ?Sized> RwLock<R, T> {
722	/// Attempts to acquire this `RwLock` with shared read access until a timeout
723	/// is reached.
724	///
725	/// If the access could not be granted before the timeout expires, then
726	/// `None` is returned. Otherwise, an RAII guard is returned which will
727	/// release the shared access when it is dropped.
728	#[inline]
729	pub fn try_read_for(&self, timeout: R::Duration) -> Option<RwLockReadGuard<'_, R, T>> {
730	if self.raw.try_lock_shared_for(timeout) {
731	// SAFETY: The lock is held, as required.
732	Some(unsafe { self.make_read_guard_unchecked() })
733	} else {
734	None
735	}
736	}
737
738	/// Attempts to acquire this `RwLock` with shared read access until a timeout
739	/// is reached.
740	///
741	/// If the access could not be granted before the timeout expires, then
742	/// `None` is returned. Otherwise, an RAII guard is returned which will
743	/// release the shared access when it is dropped.
744	#[inline]
745	pub fn try_read_until(&self, timeout: R::Instant) -> Option<RwLockReadGuard<'_, R, T>> {
746	if self.raw.try_lock_shared_until(timeout) {
747	// SAFETY: The lock is held, as required.
748	Some(unsafe { self.make_read_guard_unchecked() })
749	} else {
750	None
751	}
752	}
753
754	/// Attempts to acquire this `RwLock` with exclusive write access until a
755	/// timeout is reached.
756	///
757	/// If the access could not be granted before the timeout expires, then
758	/// `None` is returned. Otherwise, an RAII guard is returned which will
759	/// release the exclusive access when it is dropped.
760	#[inline]
761	pub fn try_write_for(&self, timeout: R::Duration) -> Option<RwLockWriteGuard<'_, R, T>> {
762	if self.raw.try_lock_exclusive_for(timeout) {
763	// SAFETY: The lock is held, as required.
764	Some(unsafe { self.make_write_guard_unchecked() })
765	} else {
766	None
767	}
768	}
769
770	/// Attempts to acquire this `RwLock` with exclusive write access until a
771	/// timeout is reached.
772	///
773	/// If the access could not be granted before the timeout expires, then
774	/// `None` is returned. Otherwise, an RAII guard is returned which will
775	/// release the exclusive access when it is dropped.
776	#[inline]
777	pub fn try_write_until(&self, timeout: R::Instant) -> Option<RwLockWriteGuard<'_, R, T>> {
778	if self.raw.try_lock_exclusive_until(timeout) {
779	// SAFETY: The lock is held, as required.
780	Some(unsafe { self.make_write_guard_unchecked() })
781	} else {
782	None
783	}
784	}
785
786	/// Attempts to acquire this `RwLock` with read access until a timeout is reached, through an `Arc`.
787	///
788	/// This method is similar to the `try_read_for` method; however, it requires the `RwLock` to be inside of an
789	/// `Arc` and the resulting read guard has no lifetime requirements.
790	#[cfg(feature = "arc_lock")]
791	#[inline]
792	pub fn try_read_arc_for(
793	self: &Arc<Self>,
794	timeout: R::Duration,
795	) -> Option<ArcRwLockReadGuard<R, T>> {
796	if self.raw.try_lock_shared_for(timeout) {
797	// SAFETY: locking guarantee is upheld
798	Some(unsafe { self.make_arc_read_guard_unchecked() })
799	} else {
800	None
801	}
802	}
803
804	/// Attempts to acquire this `RwLock` with read access until a timeout is reached, through an `Arc`.
805	///
806	/// This method is similar to the `try_read_until` method; however, it requires the `RwLock` to be inside of
807	/// an `Arc` and the resulting read guard has no lifetime requirements.
808	#[cfg(feature = "arc_lock")]
809	#[inline]
810	pub fn try_read_arc_until(
811	self: &Arc<Self>,
812	timeout: R::Instant,
813	) -> Option<ArcRwLockReadGuard<R, T>> {
814	if self.raw.try_lock_shared_until(timeout) {
815	// SAFETY: locking guarantee is upheld
816	Some(unsafe { self.make_arc_read_guard_unchecked() })
817	} else {
818	None
819	}
820	}
821
822	/// Attempts to acquire this `RwLock` with write access until a timeout is reached, through an `Arc`.
823	///
824	/// This method is similar to the `try_write_for` method; however, it requires the `RwLock` to be inside of
825	/// an `Arc` and the resulting write guard has no lifetime requirements.
826	#[cfg(feature = "arc_lock")]
827	#[inline]
828	pub fn try_write_arc_for(
829	self: &Arc<Self>,
830	timeout: R::Duration,
831	) -> Option<ArcRwLockWriteGuard<R, T>> {
832	if self.raw.try_lock_exclusive_for(timeout) {
833	// SAFETY: locking guarantee is upheld
834	Some(unsafe { self.make_arc_write_guard_unchecked() })
835	} else {
836	None
837	}
838	}
839
840	/// Attempts to acquire this `RwLock` with read access until a timeout is reached, through an `Arc`.
841	///
842	/// This method is similar to the `try_write_until` method; however, it requires the `RwLock` to be inside of
843	/// an `Arc` and the resulting read guard has no lifetime requirements.
844	#[cfg(feature = "arc_lock")]
845	#[inline]
846	pub fn try_write_arc_until(
847	self: &Arc<Self>,
848	timeout: R::Instant,
849	) -> Option<ArcRwLockWriteGuard<R, T>> {
850	if self.raw.try_lock_exclusive_until(timeout) {
851	// SAFETY: locking guarantee is upheld
852	Some(unsafe { self.make_arc_write_guard_unchecked() })
853	} else {
854	None
855	}
856	}
857	}
858
859	impl<R: RawRwLockRecursive, T: ?Sized> RwLock<R, T> {
860	/// Locks this `RwLock` with shared read access, blocking the current thread
861	/// until it can be acquired.
862	///
863	/// The calling thread will be blocked until there are no more writers which
864	/// hold the lock. There may be other readers currently inside the lock when
865	/// this method returns.
866	///
867	/// Unlike `read`, this method is guaranteed to succeed without blocking if
868	/// another read lock is held at the time of the call. This allows a thread
869	/// to recursively lock a `RwLock`. However using this method can cause
870	/// writers to starve since readers no longer block if a writer is waiting
871	/// for the lock.
872	///
873	/// Returns an RAII guard which will release this thread's shared access
874	/// once it is dropped.
875	#[inline]
876	pub fn read_recursive(&self) -> RwLockReadGuard<'_, R, T> {
877	self.raw.lock_shared_recursive();
878	// SAFETY: The lock is held, as required.
879	unsafe { self.make_read_guard_unchecked() }
880	}
881
882	/// Attempts to acquire this `RwLock` with shared read access.
883	///
884	/// If the access could not be granted at this time, then `None` is returned.
885	/// Otherwise, an RAII guard is returned which will release the shared access
886	/// when it is dropped.
887	///
888	/// This method is guaranteed to succeed if another read lock is held at the
889	/// time of the call. See the documentation for `read_recursive` for details.
890	///
891	/// This function does not block.
892	#[inline]
893	pub fn try_read_recursive(&self) -> Option<RwLockReadGuard<'_, R, T>> {
894	if self.raw.try_lock_shared_recursive() {
895	// SAFETY: The lock is held, as required.
896	Some(unsafe { self.make_read_guard_unchecked() })
897	} else {
898	None
899	}
900	}
901
902	/// Locks this `RwLock` with shared read access, through an `Arc`.
903	///
904	/// This method is similar to the `read_recursive` method; however, it requires the `RwLock` to be inside of
905	/// an `Arc` and the resulting read guard has no lifetime requirements.
906	#[cfg(feature = "arc_lock")]
907	#[inline]
908	pub fn read_arc_recursive(self: &Arc<Self>) -> ArcRwLockReadGuard<R, T> {
909	self.raw.lock_shared_recursive();
910	// SAFETY: locking guarantee is upheld
911	unsafe { self.make_arc_read_guard_unchecked() }
912	}
913
914	/// Attempts to lock this `RwLock` with shared read access, through an `Arc`.
915	///
916	/// This method is similar to the `try_read_recursive` method; however, it requires the `RwLock` to be inside
917	/// of an `Arc` and the resulting read guard has no lifetime requirements.
918	#[cfg(feature = "arc_lock")]
919	#[inline]
920	pub fn try_read_recursive_arc(self: &Arc<Self>) -> Option<ArcRwLockReadGuard<R, T>> {
921	if self.raw.try_lock_shared_recursive() {
922	// SAFETY: locking guarantee is upheld
923	Some(unsafe { self.make_arc_read_guard_unchecked() })
924	} else {
925	None
926	}
927	}
928	}
929
930	impl<R: RawRwLockRecursiveTimed, T: ?Sized> RwLock<R, T> {
931	/// Attempts to acquire this `RwLock` with shared read access until a timeout
932	/// is reached.
933	///
934	/// If the access could not be granted before the timeout expires, then
935	/// `None` is returned. Otherwise, an RAII guard is returned which will
936	/// release the shared access when it is dropped.
937	///
938	/// This method is guaranteed to succeed without blocking if another read
939	/// lock is held at the time of the call. See the documentation for
940	/// `read_recursive` for details.
941	#[inline]
942	pub fn try_read_recursive_for(
943	&self,
944	timeout: R::Duration,
945	) -> Option<RwLockReadGuard<'_, R, T>> {
946	if self.raw.try_lock_shared_recursive_for(timeout) {
947	// SAFETY: The lock is held, as required.
948	Some(unsafe { self.make_read_guard_unchecked() })
949	} else {
950	None
951	}
952	}
953
954	/// Attempts to acquire this `RwLock` with shared read access until a timeout
955	/// is reached.
956	///
957	/// If the access could not be granted before the timeout expires, then
958	/// `None` is returned. Otherwise, an RAII guard is returned which will
959	/// release the shared access when it is dropped.
960	#[inline]
961	pub fn try_read_recursive_until(
962	&self,
963	timeout: R::Instant,
964	) -> Option<RwLockReadGuard<'_, R, T>> {
965	if self.raw.try_lock_shared_recursive_until(timeout) {
966	// SAFETY: The lock is held, as required.
967	Some(unsafe { self.make_read_guard_unchecked() })
968	} else {
969	None
970	}
971	}
972
973	/// Attempts to lock this `RwLock` with read access until a timeout is reached, through an `Arc`.
974	///
975	/// This method is similar to the `try_read_recursive_for` method; however, it requires the `RwLock` to be
976	/// inside of an `Arc` and the resulting read guard has no lifetime requirements.
977	#[cfg(feature = "arc_lock")]
978	#[inline]
979	pub fn try_read_arc_recursive_for(
980	self: &Arc<Self>,
981	timeout: R::Duration,
982	) -> Option<ArcRwLockReadGuard<R, T>> {
983	if self.raw.try_lock_shared_recursive_for(timeout) {
984	// SAFETY: locking guarantee is upheld
985	Some(unsafe { self.make_arc_read_guard_unchecked() })
986	} else {
987	None
988	}
989	}
990
991	/// Attempts to lock this `RwLock` with read access until a timeout is reached, through an `Arc`.
992	///
993	/// This method is similar to the `try_read_recursive_until` method; however, it requires the `RwLock` to be
994	/// inside of an `Arc` and the resulting read guard has no lifetime requirements.
995	#[cfg(feature = "arc_lock")]
996	#[inline]
997	pub fn try_read_arc_recursive_until(
998	self: &Arc<Self>,
999	timeout: R::Instant,
1000	) -> Option<ArcRwLockReadGuard<R, T>> {
1001	if self.raw.try_lock_shared_recursive_until(timeout) {
1002	// SAFETY: locking guarantee is upheld
1003	Some(unsafe { self.make_arc_read_guard_unchecked() })
1004	} else {
1005	None
1006	}
1007	}
1008	}
1009
1010	impl<R: RawRwLockUpgrade, T: ?Sized> RwLock<R, T> {
1011	/// Creates a new `RwLockUpgradableReadGuard` without checking if the lock is held.
1012	///
1013	/// # Safety
1014	///
1015	/// This method must only be called if the thread logically holds an upgradable read lock.
1016	///
1017	/// This function does not increment the read count of the lock. Calling this function when a
1018	/// guard has already been produced is undefined behaviour unless the guard was forgotten
1019	/// with `mem::forget`.`
1020	#[inline]
1021	pub unsafe fn make_upgradable_guard_unchecked(&self) -> RwLockUpgradableReadGuard<'_, R, T> {
1022	RwLockUpgradableReadGuard {
1023	rwlock: self,
1024	marker: PhantomData,
1025	}
1026	}
1027
1028	/// Locks this `RwLock` with upgradable read access, blocking the current thread
1029	/// until it can be acquired.
1030	///
1031	/// The calling thread will be blocked until there are no more writers or other
1032	/// upgradable reads which hold the lock. There may be other readers currently
1033	/// inside the lock when this method returns.
1034	///
1035	/// Returns an RAII guard which will release this thread's shared access
1036	/// once it is dropped.
1037	#[inline]
1038	pub fn upgradable_read(&self) -> RwLockUpgradableReadGuard<'_, R, T> {
1039	self.raw.lock_upgradable();
1040	// SAFETY: The lock is held, as required.
1041	unsafe { self.make_upgradable_guard_unchecked() }
1042	}
1043
1044	/// Attempts to acquire this `RwLock` with upgradable read access.
1045	///
1046	/// If the access could not be granted at this time, then `None` is returned.
1047	/// Otherwise, an RAII guard is returned which will release the shared access
1048	/// when it is dropped.
1049	///
1050	/// This function does not block.
1051	#[inline]
1052	pub fn try_upgradable_read(&self) -> Option<RwLockUpgradableReadGuard<'_, R, T>> {
1053	if self.raw.try_lock_upgradable() {
1054	// SAFETY: The lock is held, as required.
1055	Some(unsafe { self.make_upgradable_guard_unchecked() })
1056	} else {
1057	None
1058	}
1059	}
1060
1061	/// Creates a new `ArcRwLockUpgradableReadGuard` without checking if the lock is held.
1062	///
1063	/// # Safety
1064	///
1065	/// This method must only be called if the thread logically holds an upgradable read lock.
1066	///
1067	/// This function does not increment the read count of the lock. Calling this function when a
1068	/// guard has already been produced is undefined behaviour unless the guard was forgotten
1069	/// with `mem::forget`.`
1070	#[cfg(feature = "arc_lock")]
1071	#[inline]
1072	pub unsafe fn make_upgradable_arc_guard_unchecked(
1073	self: &Arc<Self>,
1074	) -> ArcRwLockUpgradableReadGuard<R, T> {
1075	ArcRwLockUpgradableReadGuard {
1076	rwlock: self.clone(),
1077	marker: PhantomData,
1078	}
1079	}
1080
1081	/// Locks this `RwLock` with upgradable read access, through an `Arc`.
1082	///
1083	/// This method is similar to the `upgradable_read` method; however, it requires the `RwLock` to be
1084	/// inside of an `Arc` and the resulting read guard has no lifetime requirements.
1085	#[cfg(feature = "arc_lock")]
1086	#[inline]
1087	pub fn upgradable_read_arc(self: &Arc<Self>) -> ArcRwLockUpgradableReadGuard<R, T> {
1088	self.raw.lock_upgradable();
1089	// SAFETY: locking guarantee is upheld
1090	unsafe { self.make_upgradable_arc_guard_unchecked() }
1091	}
1092
1093	/// Attempts to lock this `RwLock` with upgradable read access, through an `Arc`.
1094	///
1095	/// This method is similar to the `try_upgradable_read` method; however, it requires the `RwLock` to be
1096	/// inside of an `Arc` and the resulting read guard has no lifetime requirements.
1097	#[cfg(feature = "arc_lock")]
1098	#[inline]
1099	pub fn try_upgradable_read_arc(self: &Arc<Self>) -> Option<ArcRwLockUpgradableReadGuard<R, T>> {
1100	if self.raw.try_lock_upgradable() {
1101	// SAFETY: locking guarantee is upheld
1102	Some(unsafe { self.make_upgradable_arc_guard_unchecked() })
1103	} else {
1104	None
1105	}
1106	}
1107	}
1108
1109	impl<R: RawRwLockUpgradeTimed, T: ?Sized> RwLock<R, T> {
1110	/// Attempts to acquire this `RwLock` with upgradable read access until a timeout
1111	/// is reached.
1112	///
1113	/// If the access could not be granted before the timeout expires, then
1114	/// `None` is returned. Otherwise, an RAII guard is returned which will
1115	/// release the shared access when it is dropped.
1116	#[inline]
1117	pub fn try_upgradable_read_for(
1118	&self,
1119	timeout: R::Duration,
1120	) -> Option<RwLockUpgradableReadGuard<'_, R, T>> {
1121	if self.raw.try_lock_upgradable_for(timeout) {
1122	// SAFETY: The lock is held, as required.
1123	Some(unsafe { self.make_upgradable_guard_unchecked() })
1124	} else {
1125	None
1126	}
1127	}
1128
1129	/// Attempts to acquire this `RwLock` with upgradable read access until a timeout
1130	/// is reached.
1131	///
1132	/// If the access could not be granted before the timeout expires, then
1133	/// `None` is returned. Otherwise, an RAII guard is returned which will
1134	/// release the shared access when it is dropped.
1135	#[inline]
1136	pub fn try_upgradable_read_until(
1137	&self,
1138	timeout: R::Instant,
1139	) -> Option<RwLockUpgradableReadGuard<'_, R, T>> {
1140	if self.raw.try_lock_upgradable_until(timeout) {
1141	// SAFETY: The lock is held, as required.
1142	Some(unsafe { self.make_upgradable_guard_unchecked() })
1143	} else {
1144	None
1145	}
1146	}
1147
1148	/// Attempts to lock this `RwLock` with upgradable access until a timeout is reached, through an `Arc`.
1149	///
1150	/// This method is similar to the `try_upgradable_read_for` method; however, it requires the `RwLock` to be
1151	/// inside of an `Arc` and the resulting read guard has no lifetime requirements.
1152	#[cfg(feature = "arc_lock")]
1153	#[inline]
1154	pub fn try_upgradable_read_arc_for(
1155	self: &Arc<Self>,
1156	timeout: R::Duration,
1157	) -> Option<ArcRwLockUpgradableReadGuard<R, T>> {
1158	if self.raw.try_lock_upgradable_for(timeout) {
1159	// SAFETY: locking guarantee is upheld
1160	Some(unsafe { self.make_upgradable_arc_guard_unchecked() })
1161	} else {
1162	None
1163	}
1164	}
1165
1166	/// Attempts to lock this `RwLock` with upgradable access until a timeout is reached, through an `Arc`.
1167	///
1168	/// This method is similar to the `try_upgradable_read_until` method; however, it requires the `RwLock` to be
1169	/// inside of an `Arc` and the resulting read guard has no lifetime requirements.
1170	#[cfg(feature = "arc_lock")]
1171	#[inline]
1172	pub fn try_upgradable_read_arc_until(
1173	self: &Arc<Self>,
1174	timeout: R::Instant,
1175	) -> Option<ArcRwLockUpgradableReadGuard<R, T>> {
1176	if self.raw.try_lock_upgradable_until(timeout) {
1177	// SAFETY: locking guarantee is upheld
1178	Some(unsafe { self.make_upgradable_arc_guard_unchecked() })
1179	} else {
1180	None
1181	}
1182	}
1183	}
1184
1185	impl<R: RawRwLock, T: ?Sized + Default> Default for RwLock<R, T> {
1186	#[inline]
1187	fn default() -> RwLock<R, T> {
1188	RwLock::new(Default::default())
1189	}
1190	}
1191
1192	impl<R: RawRwLock, T> From<T> for RwLock<R, T> {
1193	#[inline]
1194	fn from(t: T) -> RwLock<R, T> {
1195	RwLock::new(t)
1196	}
1197	}
1198
1199	impl<R: RawRwLock, T: ?Sized + fmt::Debug> fmt::Debug for RwLock<R, T> {
1200	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1201	match self.try_read() {
1202	Some(guard) => f.debug_struct("RwLock").field("data", &&*guard).finish(),
1203	None => {
1204	struct LockedPlaceholder;
1205	impl fmt::Debug for LockedPlaceholder {
1206	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1207	f.write_str("<locked>")
1208	}
1209	}
1210
1211	f.debug_struct("RwLock")
1212	.field("data", &LockedPlaceholder)
1213	.finish()
1214	}
1215	}
1216	}
1217	}
1218
1219	/// RAII structure used to release the shared read access of a lock when
1220	/// dropped.
1221	#[clippy::has_significant_drop]
1222	#[must_use = "if unused the RwLock will immediately unlock"]
1223	pub struct RwLockReadGuard<'a, R: RawRwLock, T: ?Sized> {
1224	rwlock: &'a RwLock<R, T>,
1225	marker: PhantomData<(&'a T, R::GuardMarker)>,
1226	}
1227
1228	unsafe impl<R: RawRwLock + Sync, T: Sync + ?Sized> Sync for RwLockReadGuard<'_, R, T> {}
1229
1230	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> RwLockReadGuard<'a, R, T> {
1231	/// Returns a reference to the original reader-writer lock object.
1232	pub fn rwlock(s: &Self) -> &'a RwLock<R, T> {
1233	s.rwlock
1234	}
1235
1236	/// Make a new `MappedRwLockReadGuard` for a component of the locked data.
1237	///
1238	/// This operation cannot fail as the `RwLockReadGuard` passed
1239	/// in already locked the data.
1240	///
1241	/// This is an associated function that needs to be
1242	/// used as `RwLockReadGuard::map(...)`. A method would interfere with methods of
1243	/// the same name on the contents of the locked data.
1244	#[inline]
1245	pub fn map<U: ?Sized, F>(s: Self, f: F) -> MappedRwLockReadGuard<'a, R, U>
1246	where
1247	F: FnOnce(&T) -> &U,
1248	{
1249	let raw = &s.rwlock.raw;
1250	let data = f(unsafe { &*s.rwlock.data.get() });
1251	mem::forget(s);
1252	MappedRwLockReadGuard {
1253	raw,
1254	data,
1255	marker: PhantomData,
1256	}
1257	}
1258
1259	/// Attempts to make a new `MappedRwLockReadGuard` for a component of the
1260	/// locked data. Returns the original guard if the closure returns `None`.
1261	///
1262	/// This operation cannot fail as the `RwLockReadGuard` passed
1263	/// in already locked the data.
1264	///
1265	/// This is an associated function that needs to be
1266	/// used as `RwLockReadGuard::try_map(...)`. A method would interfere with methods of
1267	/// the same name on the contents of the locked data.
1268	#[inline]
1269	pub fn try_map<U: ?Sized, F>(s: Self, f: F) -> Result<MappedRwLockReadGuard<'a, R, U>, Self>
1270	where
1271	F: FnOnce(&T) -> Option<&U>,
1272	{
1273	let raw = &s.rwlock.raw;
1274	let data = match f(unsafe { &*s.rwlock.data.get() }) {
1275	Some(data) => data,
1276	None => return Err(s),
1277	};
1278	mem::forget(s);
1279	Ok(MappedRwLockReadGuard {
1280	raw,
1281	data,
1282	marker: PhantomData,
1283	})
1284	}
1285
1286	/// Temporarily unlocks the `RwLock` to execute the given function.
1287	///
1288	/// This is safe because `&mut` guarantees that there exist no other
1289	/// references to the data protected by the `RwLock`.
1290	#[inline]
1291	pub fn unlocked<F, U>(s: &mut Self, f: F) -> U
1292	where
1293	F: FnOnce() -> U,
1294	{
1295	// Safety: An RwLockReadGuard always holds a shared lock.
1296	unsafe {
1297	s.rwlock.raw.unlock_shared();
1298	}
1299	defer!(s.rwlock.raw.lock_shared());
1300	f()
1301	}
1302	}
1303
1304	impl<'a, R: RawRwLockFair + 'a, T: ?Sized + 'a> RwLockReadGuard<'a, R, T> {
1305	/// Unlocks the `RwLock` using a fair unlock protocol.
1306	///
1307	/// By default, `RwLock` is unfair and allow the current thread to re-lock
1308	/// the `RwLock` before another has the chance to acquire the lock, even if
1309	/// that thread has been blocked on the `RwLock` for a long time. This is
1310	/// the default because it allows much higher throughput as it avoids
1311	/// forcing a context switch on every `RwLock` unlock. This can result in one
1312	/// thread acquiring a `RwLock` many more times than other threads.
1313	///
1314	/// However in some cases it can be beneficial to ensure fairness by forcing
1315	/// the lock to pass on to a waiting thread if there is one. This is done by
1316	/// using this method instead of dropping the `RwLockReadGuard` normally.
1317	#[inline]
1318	pub fn unlock_fair(s: Self) {
1319	// Safety: An RwLockReadGuard always holds a shared lock.
1320	unsafe {
1321	s.rwlock.raw.unlock_shared_fair();
1322	}
1323	mem::forget(s);
1324	}
1325
1326	/// Temporarily unlocks the `RwLock` to execute the given function.
1327	///
1328	/// The `RwLock` is unlocked a fair unlock protocol.
1329	///
1330	/// This is safe because `&mut` guarantees that there exist no other
1331	/// references to the data protected by the `RwLock`.
1332	#[inline]
1333	pub fn unlocked_fair<F, U>(s: &mut Self, f: F) -> U
1334	where
1335	F: FnOnce() -> U,
1336	{
1337	// Safety: An RwLockReadGuard always holds a shared lock.
1338	unsafe {
1339	s.rwlock.raw.unlock_shared_fair();
1340	}
1341	defer!(s.rwlock.raw.lock_shared());
1342	f()
1343	}
1344
1345	/// Temporarily yields the `RwLock` to a waiting thread if there is one.
1346	///
1347	/// This method is functionally equivalent to calling `unlock_fair` followed
1348	/// by `read`, however it can be much more efficient in the case where there
1349	/// are no waiting threads.
1350	#[inline]
1351	pub fn bump(s: &mut Self) {
1352	// Safety: An RwLockReadGuard always holds a shared lock.
1353	unsafe {
1354	s.rwlock.raw.bump_shared();
1355	}
1356	}
1357	}
1358
1359	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Deref for RwLockReadGuard<'a, R, T> {
1360	type Target = T;
1361	#[inline]
1362	fn deref(&self) -> &T {
1363	unsafe { &*self.rwlock.data.get() }
1364	}
1365	}
1366
1367	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Drop for RwLockReadGuard<'a, R, T> {
1368	#[inline]
1369	fn drop(&mut self) {
1370	// Safety: An RwLockReadGuard always holds a shared lock.
1371	unsafe {
1372	self.rwlock.raw.unlock_shared();
1373	}
1374	}
1375	}
1376
1377	impl<'a, R: RawRwLock + 'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for RwLockReadGuard<'a, R, T> {
1378	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1379	fmt::Debug::fmt(&**self, f)
1380	}
1381	}
1382
1383	impl<'a, R: RawRwLock + 'a, T: fmt::Display + ?Sized + 'a> fmt::Display
1384	for RwLockReadGuard<'a, R, T>
1385	{
1386	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1387	(**self).fmt(f)
1388	}
1389	}
1390
1391	#[cfg(feature = "owning_ref")]
1392	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> StableAddress for RwLockReadGuard<'a, R, T> {}
1393
1394	/// An RAII rwlock guard returned by the `Arc` locking operations on `RwLock`.
1395	///
1396	/// This is similar to the `RwLockReadGuard` struct, except instead of using a reference to unlock the `RwLock`
1397	/// it uses an `Arc<RwLock>`. This has several advantages, most notably that it has an `'static` lifetime.
1398	#[cfg(feature = "arc_lock")]
1399	#[clippy::has_significant_drop]
1400	#[must_use = "if unused the RwLock will immediately unlock"]
1401	pub struct ArcRwLockReadGuard<R: RawRwLock, T: ?Sized> {
1402	rwlock: Arc<RwLock<R, T>>,
1403	marker: PhantomData<R::GuardMarker>,
1404	}
1405
1406	#[cfg(feature = "arc_lock")]
1407	impl<R: RawRwLock, T: ?Sized> ArcRwLockReadGuard<R, T> {
1408	/// Returns a reference to the rwlock, contained in its `Arc`.
1409	pub fn rwlock(s: &Self) -> &Arc<RwLock<R, T>> {
1410	&s.rwlock
1411	}
1412
1413	/// Temporarily unlocks the `RwLock` to execute the given function.
1414	///
1415	/// This is functionally identical to the `unlocked` method on [`RwLockReadGuard`].
1416	#[inline]
1417	pub fn unlocked<F, U>(s: &mut Self, f: F) -> U
1418	where
1419	F: FnOnce() -> U,
1420	{
1421	// Safety: An RwLockReadGuard always holds a shared lock.
1422	unsafe {
1423	s.rwlock.raw.unlock_shared();
1424	}
1425	defer!(s.rwlock.raw.lock_shared());
1426	f()
1427	}
1428	}
1429
1430	#[cfg(feature = "arc_lock")]
1431	impl<R: RawRwLockFair, T: ?Sized> ArcRwLockReadGuard<R, T> {
1432	/// Unlocks the `RwLock` using a fair unlock protocol.
1433	///
1434	/// This is functionally identical to the `unlock_fair` method on [`RwLockReadGuard`].
1435	#[inline]
1436	pub fn unlock_fair(s: Self) {
1437	// Safety: An RwLockReadGuard always holds a shared lock.
1438	unsafe {
1439	s.rwlock.raw.unlock_shared_fair();
1440	}
1441
1442	// SAFETY: ensure the Arc has its refcount decremented
1443	let mut s = ManuallyDrop::new(s);
1444	unsafe { ptr::drop_in_place(&mut s.rwlock) };
1445	}
1446
1447	/// Temporarily unlocks the `RwLock` to execute the given function.
1448	///
1449	/// This is functionally identical to the `unlocked_fair` method on [`RwLockReadGuard`].
1450	#[inline]
1451	pub fn unlocked_fair<F, U>(s: &mut Self, f: F) -> U
1452	where
1453	F: FnOnce() -> U,
1454	{
1455	// Safety: An RwLockReadGuard always holds a shared lock.
1456	unsafe {
1457	s.rwlock.raw.unlock_shared_fair();
1458	}
1459	defer!(s.rwlock.raw.lock_shared());
1460	f()
1461	}
1462
1463	/// Temporarily yields the `RwLock` to a waiting thread if there is one.
1464	///
1465	/// This is functionally identical to the `bump` method on [`RwLockReadGuard`].
1466	#[inline]
1467	pub fn bump(s: &mut Self) {
1468	// Safety: An RwLockReadGuard always holds a shared lock.
1469	unsafe {
1470	s.rwlock.raw.bump_shared();
1471	}
1472	}
1473	}
1474
1475	#[cfg(feature = "arc_lock")]
1476	impl<R: RawRwLock, T: ?Sized> Deref for ArcRwLockReadGuard<R, T> {
1477	type Target = T;
1478	#[inline]
1479	fn deref(&self) -> &T {
1480	unsafe { &*self.rwlock.data.get() }
1481	}
1482	}
1483
1484	#[cfg(feature = "arc_lock")]
1485	impl<R: RawRwLock, T: ?Sized> Drop for ArcRwLockReadGuard<R, T> {
1486	#[inline]
1487	fn drop(&mut self) {
1488	// Safety: An RwLockReadGuard always holds a shared lock.
1489	unsafe {
1490	self.rwlock.raw.unlock_shared();
1491	}
1492	}
1493	}
1494
1495	#[cfg(feature = "arc_lock")]
1496	impl<R: RawRwLock, T: fmt::Debug + ?Sized> fmt::Debug for ArcRwLockReadGuard<R, T> {
1497	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1498	fmt::Debug::fmt(&**self, f)
1499	}
1500	}
1501
1502	#[cfg(feature = "arc_lock")]
1503	impl<R: RawRwLock, T: fmt::Display + ?Sized> fmt::Display for ArcRwLockReadGuard<R, T> {
1504	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1505	(**self).fmt(f)
1506	}
1507	}
1508
1509	/// RAII structure used to release the exclusive write access of a lock when
1510	/// dropped.
1511	#[clippy::has_significant_drop]
1512	#[must_use = "if unused the RwLock will immediately unlock"]
1513	pub struct RwLockWriteGuard<'a, R: RawRwLock, T: ?Sized> {
1514	rwlock: &'a RwLock<R, T>,
1515	marker: PhantomData<(&'a mut T, R::GuardMarker)>,
1516	}
1517
1518	unsafe impl<R: RawRwLock + Sync, T: Sync + ?Sized> Sync for RwLockWriteGuard<'_, R, T> {}
1519
1520	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> RwLockWriteGuard<'a, R, T> {
1521	/// Returns a reference to the original reader-writer lock object.
1522	pub fn rwlock(s: &Self) -> &'a RwLock<R, T> {
1523	s.rwlock
1524	}
1525
1526	/// Make a new `MappedRwLockWriteGuard` for a component of the locked data.
1527	///
1528	/// This operation cannot fail as the `RwLockWriteGuard` passed
1529	/// in already locked the data.
1530	///
1531	/// This is an associated function that needs to be
1532	/// used as `RwLockWriteGuard::map(...)`. A method would interfere with methods of
1533	/// the same name on the contents of the locked data.
1534	#[inline]
1535	pub fn map<U: ?Sized, F>(s: Self, f: F) -> MappedRwLockWriteGuard<'a, R, U>
1536	where
1537	F: FnOnce(&mut T) -> &mut U,
1538	{
1539	let raw = &s.rwlock.raw;
1540	let data = f(unsafe { &mut *s.rwlock.data.get() });
1541	mem::forget(s);
1542	MappedRwLockWriteGuard {
1543	raw,
1544	data,
1545	marker: PhantomData,
1546	}
1547	}
1548
1549	/// Attempts to make a new `MappedRwLockWriteGuard` for a component of the
1550	/// locked data. The original guard is return if the closure returns `None`.
1551	///
1552	/// This operation cannot fail as the `RwLockWriteGuard` passed
1553	/// in already locked the data.
1554	///
1555	/// This is an associated function that needs to be
1556	/// used as `RwLockWriteGuard::try_map(...)`. A method would interfere with methods of
1557	/// the same name on the contents of the locked data.
1558	#[inline]
1559	pub fn try_map<U: ?Sized, F>(s: Self, f: F) -> Result<MappedRwLockWriteGuard<'a, R, U>, Self>
1560	where
1561	F: FnOnce(&mut T) -> Option<&mut U>,
1562	{
1563	let raw = &s.rwlock.raw;
1564	let data = match f(unsafe { &mut *s.rwlock.data.get() }) {
1565	Some(data) => data,
1566	None => return Err(s),
1567	};
1568	mem::forget(s);
1569	Ok(MappedRwLockWriteGuard {
1570	raw,
1571	data,
1572	marker: PhantomData,
1573	})
1574	}
1575
1576	/// Temporarily unlocks the `RwLock` to execute the given function.
1577	///
1578	/// This is safe because `&mut` guarantees that there exist no other
1579	/// references to the data protected by the `RwLock`.
1580	#[inline]
1581	pub fn unlocked<F, U>(s: &mut Self, f: F) -> U
1582	where
1583	F: FnOnce() -> U,
1584	{
1585	// Safety: An RwLockReadGuard always holds a shared lock.
1586	unsafe {
1587	s.rwlock.raw.unlock_exclusive();
1588	}
1589	defer!(s.rwlock.raw.lock_exclusive());
1590	f()
1591	}
1592	}
1593
1594	impl<'a, R: RawRwLockDowngrade + 'a, T: ?Sized + 'a> RwLockWriteGuard<'a, R, T> {
1595	/// Atomically downgrades a write lock into a read lock without allowing any
1596	/// writers to take exclusive access of the lock in the meantime.
1597	///
1598	/// Note that if there are any writers currently waiting to take the lock
1599	/// then other readers may not be able to acquire the lock even if it was
1600	/// downgraded.
1601	pub fn downgrade(s: Self) -> RwLockReadGuard<'a, R, T> {
1602	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1603	unsafe {
1604	s.rwlock.raw.downgrade();
1605	}
1606	let rwlock = s.rwlock;
1607	mem::forget(s);
1608	RwLockReadGuard {
1609	rwlock,
1610	marker: PhantomData,
1611	}
1612	}
1613	}
1614
1615	impl<'a, R: RawRwLockUpgradeDowngrade + 'a, T: ?Sized + 'a> RwLockWriteGuard<'a, R, T> {
1616	/// Atomically downgrades a write lock into an upgradable read lock without allowing any
1617	/// writers to take exclusive access of the lock in the meantime.
1618	///
1619	/// Note that if there are any writers currently waiting to take the lock
1620	/// then other readers may not be able to acquire the lock even if it was
1621	/// downgraded.
1622	pub fn downgrade_to_upgradable(s: Self) -> RwLockUpgradableReadGuard<'a, R, T> {
1623	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1624	unsafe {
1625	s.rwlock.raw.downgrade_to_upgradable();
1626	}
1627	let rwlock = s.rwlock;
1628	mem::forget(s);
1629	RwLockUpgradableReadGuard {
1630	rwlock,
1631	marker: PhantomData,
1632	}
1633	}
1634	}
1635
1636	impl<'a, R: RawRwLockFair + 'a, T: ?Sized + 'a> RwLockWriteGuard<'a, R, T> {
1637	/// Unlocks the `RwLock` using a fair unlock protocol.
1638	///
1639	/// By default, `RwLock` is unfair and allow the current thread to re-lock
1640	/// the `RwLock` before another has the chance to acquire the lock, even if
1641	/// that thread has been blocked on the `RwLock` for a long time. This is
1642	/// the default because it allows much higher throughput as it avoids
1643	/// forcing a context switch on every `RwLock` unlock. This can result in one
1644	/// thread acquiring a `RwLock` many more times than other threads.
1645	///
1646	/// However in some cases it can be beneficial to ensure fairness by forcing
1647	/// the lock to pass on to a waiting thread if there is one. This is done by
1648	/// using this method instead of dropping the `RwLockWriteGuard` normally.
1649	#[inline]
1650	pub fn unlock_fair(s: Self) {
1651	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1652	unsafe {
1653	s.rwlock.raw.unlock_exclusive_fair();
1654	}
1655	mem::forget(s);
1656	}
1657
1658	/// Temporarily unlocks the `RwLock` to execute the given function.
1659	///
1660	/// The `RwLock` is unlocked a fair unlock protocol.
1661	///
1662	/// This is safe because `&mut` guarantees that there exist no other
1663	/// references to the data protected by the `RwLock`.
1664	#[inline]
1665	pub fn unlocked_fair<F, U>(s: &mut Self, f: F) -> U
1666	where
1667	F: FnOnce() -> U,
1668	{
1669	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1670	unsafe {
1671	s.rwlock.raw.unlock_exclusive_fair();
1672	}
1673	defer!(s.rwlock.raw.lock_exclusive());
1674	f()
1675	}
1676
1677	/// Temporarily yields the `RwLock` to a waiting thread if there is one.
1678	///
1679	/// This method is functionally equivalent to calling `unlock_fair` followed
1680	/// by `write`, however it can be much more efficient in the case where there
1681	/// are no waiting threads.
1682	#[inline]
1683	pub fn bump(s: &mut Self) {
1684	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1685	unsafe {
1686	s.rwlock.raw.bump_exclusive();
1687	}
1688	}
1689	}
1690
1691	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Deref for RwLockWriteGuard<'a, R, T> {
1692	type Target = T;
1693	#[inline]
1694	fn deref(&self) -> &T {
1695	unsafe { &*self.rwlock.data.get() }
1696	}
1697	}
1698
1699	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> DerefMut for RwLockWriteGuard<'a, R, T> {
1700	#[inline]
1701	fn deref_mut(&mut self) -> &mut T {
1702	unsafe { &mut *self.rwlock.data.get() }
1703	}
1704	}
1705
1706	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Drop for RwLockWriteGuard<'a, R, T> {
1707	#[inline]
1708	fn drop(&mut self) {
1709	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1710	unsafe {
1711	self.rwlock.raw.unlock_exclusive();
1712	}
1713	}
1714	}
1715
1716	impl<'a, R: RawRwLock + 'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug for RwLockWriteGuard<'a, R, T> {
1717	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1718	fmt::Debug::fmt(&**self, f)
1719	}
1720	}
1721
1722	impl<'a, R: RawRwLock + 'a, T: fmt::Display + ?Sized + 'a> fmt::Display
1723	for RwLockWriteGuard<'a, R, T>
1724	{
1725	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1726	(**self).fmt(f)
1727	}
1728	}
1729
1730	#[cfg(feature = "owning_ref")]
1731	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> StableAddress for RwLockWriteGuard<'a, R, T> {}
1732
1733	/// An RAII rwlock guard returned by the `Arc` locking operations on `RwLock`.
1734	/// This is similar to the `RwLockWriteGuard` struct, except instead of using a reference to unlock the `RwLock`
1735	/// it uses an `Arc<RwLock>`. This has several advantages, most notably that it has an `'static` lifetime.
1736	#[cfg(feature = "arc_lock")]
1737	#[clippy::has_significant_drop]
1738	#[must_use = "if unused the RwLock will immediately unlock"]
1739	pub struct ArcRwLockWriteGuard<R: RawRwLock, T: ?Sized> {
1740	rwlock: Arc<RwLock<R, T>>,
1741	marker: PhantomData<R::GuardMarker>,
1742	}
1743
1744	#[cfg(feature = "arc_lock")]
1745	impl<R: RawRwLock, T: ?Sized> ArcRwLockWriteGuard<R, T> {
1746	/// Returns a reference to the rwlock, contained in its `Arc`.
1747	pub fn rwlock(s: &Self) -> &Arc<RwLock<R, T>> {
1748	&s.rwlock
1749	}
1750
1751	/// Temporarily unlocks the `RwLock` to execute the given function.
1752	///
1753	/// This is functionally equivalent to the `unlocked` method on [`RwLockWriteGuard`].
1754	#[inline]
1755	pub fn unlocked<F, U>(s: &mut Self, f: F) -> U
1756	where
1757	F: FnOnce() -> U,
1758	{
1759	// Safety: An RwLockWriteGuard always holds a shared lock.
1760	unsafe {
1761	s.rwlock.raw.unlock_exclusive();
1762	}
1763	defer!(s.rwlock.raw.lock_exclusive());
1764	f()
1765	}
1766	}
1767
1768	#[cfg(feature = "arc_lock")]
1769	impl<R: RawRwLockDowngrade, T: ?Sized> ArcRwLockWriteGuard<R, T> {
1770	/// Atomically downgrades a write lock into a read lock without allowing any
1771	/// writers to take exclusive access of the lock in the meantime.
1772	///
1773	/// This is functionally equivalent to the `downgrade` method on [`RwLockWriteGuard`].
1774	pub fn downgrade(s: Self) -> ArcRwLockReadGuard<R, T> {
1775	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1776	unsafe {
1777	s.rwlock.raw.downgrade();
1778	}
1779
1780	// SAFETY: prevent the arc's refcount from changing using ManuallyDrop and ptr::read
1781	let s = ManuallyDrop::new(s);
1782	let rwlock = unsafe { ptr::read(&s.rwlock) };
1783
1784	ArcRwLockReadGuard {
1785	rwlock,
1786	marker: PhantomData,
1787	}
1788	}
1789	}
1790
1791	#[cfg(feature = "arc_lock")]
1792	impl<R: RawRwLockUpgradeDowngrade, T: ?Sized> ArcRwLockWriteGuard<R, T> {
1793	/// Atomically downgrades a write lock into an upgradable read lock without allowing any
1794	/// writers to take exclusive access of the lock in the meantime.
1795	///
1796	/// This is functionally identical to the `downgrade_to_upgradable` method on [`RwLockWriteGuard`].
1797	pub fn downgrade_to_upgradable(s: Self) -> ArcRwLockUpgradableReadGuard<R, T> {
1798	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1799	unsafe {
1800	s.rwlock.raw.downgrade_to_upgradable();
1801	}
1802
1803	// SAFETY: same as above
1804	let s = ManuallyDrop::new(s);
1805	let rwlock = unsafe { ptr::read(&s.rwlock) };
1806
1807	ArcRwLockUpgradableReadGuard {
1808	rwlock,
1809	marker: PhantomData,
1810	}
1811	}
1812	}
1813
1814	#[cfg(feature = "arc_lock")]
1815	impl<R: RawRwLockFair, T: ?Sized> ArcRwLockWriteGuard<R, T> {
1816	/// Unlocks the `RwLock` using a fair unlock protocol.
1817	///
1818	/// This is functionally equivalent to the `unlock_fair` method on [`RwLockWriteGuard`].
1819	#[inline]
1820	pub fn unlock_fair(s: Self) {
1821	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1822	unsafe {
1823	s.rwlock.raw.unlock_exclusive_fair();
1824	}
1825
1826	// SAFETY: prevent the Arc from leaking memory
1827	let mut s = ManuallyDrop::new(s);
1828	unsafe { ptr::drop_in_place(&mut s.rwlock) };
1829	}
1830
1831	/// Temporarily unlocks the `RwLock` to execute the given function.
1832	///
1833	/// This is functionally equivalent to the `unlocked_fair` method on [`RwLockWriteGuard`].
1834	#[inline]
1835	pub fn unlocked_fair<F, U>(s: &mut Self, f: F) -> U
1836	where
1837	F: FnOnce() -> U,
1838	{
1839	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1840	unsafe {
1841	s.rwlock.raw.unlock_exclusive_fair();
1842	}
1843	defer!(s.rwlock.raw.lock_exclusive());
1844	f()
1845	}
1846
1847	/// Temporarily yields the `RwLock` to a waiting thread if there is one.
1848	///
1849	/// This method is functionally equivalent to the `bump` method on [`RwLockWriteGuard`].
1850	#[inline]
1851	pub fn bump(s: &mut Self) {
1852	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1853	unsafe {
1854	s.rwlock.raw.bump_exclusive();
1855	}
1856	}
1857	}
1858
1859	#[cfg(feature = "arc_lock")]
1860	impl<R: RawRwLock, T: ?Sized> Deref for ArcRwLockWriteGuard<R, T> {
1861	type Target = T;
1862	#[inline]
1863	fn deref(&self) -> &T {
1864	unsafe { &*self.rwlock.data.get() }
1865	}
1866	}
1867
1868	#[cfg(feature = "arc_lock")]
1869	impl<R: RawRwLock, T: ?Sized> DerefMut for ArcRwLockWriteGuard<R, T> {
1870	#[inline]
1871	fn deref_mut(&mut self) -> &mut T {
1872	unsafe { &mut *self.rwlock.data.get() }
1873	}
1874	}
1875
1876	#[cfg(feature = "arc_lock")]
1877	impl<R: RawRwLock, T: ?Sized> Drop for ArcRwLockWriteGuard<R, T> {
1878	#[inline]
1879	fn drop(&mut self) {
1880	// Safety: An RwLockWriteGuard always holds an exclusive lock.
1881	unsafe {
1882	self.rwlock.raw.unlock_exclusive();
1883	}
1884	}
1885	}
1886
1887	#[cfg(feature = "arc_lock")]
1888	impl<R: RawRwLock, T: fmt::Debug + ?Sized> fmt::Debug for ArcRwLockWriteGuard<R, T> {
1889	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1890	fmt::Debug::fmt(&**self, f)
1891	}
1892	}
1893
1894	#[cfg(feature = "arc_lock")]
1895	impl<R: RawRwLock, T: fmt::Display + ?Sized> fmt::Display for ArcRwLockWriteGuard<R, T> {
1896	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1897	(**self).fmt(f)
1898	}
1899	}
1900
1901	/// RAII structure used to release the upgradable read access of a lock when
1902	/// dropped.
1903	#[clippy::has_significant_drop]
1904	#[must_use = "if unused the RwLock will immediately unlock"]
1905	pub struct RwLockUpgradableReadGuard<'a, R: RawRwLockUpgrade, T: ?Sized> {
1906	rwlock: &'a RwLock<R, T>,
1907	marker: PhantomData<(&'a T, R::GuardMarker)>,
1908	}
1909
1910	unsafe impl<'a, R: RawRwLockUpgrade + 'a, T: ?Sized + Sync + 'a> Sync
1911	for RwLockUpgradableReadGuard<'a, R, T>
1912	{
1913	}
1914
1915	impl<'a, R: RawRwLockUpgrade + 'a, T: ?Sized + 'a> RwLockUpgradableReadGuard<'a, R, T> {
1916	/// Returns a reference to the original reader-writer lock object.
1917	pub fn rwlock(s: &Self) -> &'a RwLock<R, T> {
1918	s.rwlock
1919	}
1920
1921	/// Temporarily unlocks the `RwLock` to execute the given function.
1922	///
1923	/// This is safe because `&mut` guarantees that there exist no other
1924	/// references to the data protected by the `RwLock`.
1925	#[inline]
1926	pub fn unlocked<F, U>(s: &mut Self, f: F) -> U
1927	where
1928	F: FnOnce() -> U,
1929	{
1930	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
1931	unsafe {
1932	s.rwlock.raw.unlock_upgradable();
1933	}
1934	defer!(s.rwlock.raw.lock_upgradable());
1935	f()
1936	}
1937
1938	/// Atomically upgrades an upgradable read lock lock into an exclusive write lock,
1939	/// blocking the current thread until it can be acquired.
1940	pub fn upgrade(s: Self) -> RwLockWriteGuard<'a, R, T> {
1941	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
1942	unsafe {
1943	s.rwlock.raw.upgrade();
1944	}
1945	let rwlock = s.rwlock;
1946	mem::forget(s);
1947	RwLockWriteGuard {
1948	rwlock,
1949	marker: PhantomData,
1950	}
1951	}
1952
1953	/// Tries to atomically upgrade an upgradable read lock into an exclusive write lock.
1954	///
1955	/// If the access could not be granted at this time, then the current guard is returned.
1956	pub fn try_upgrade(s: Self) -> Result<RwLockWriteGuard<'a, R, T>, Self> {
1957	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
1958	if unsafe { s.rwlock.raw.try_upgrade() } {
1959	let rwlock = s.rwlock;
1960	mem::forget(s);
1961	Ok(RwLockWriteGuard {
1962	rwlock,
1963	marker: PhantomData,
1964	})
1965	} else {
1966	Err(s)
1967	}
1968	}
1969	}
1970
1971	impl<'a, R: RawRwLockUpgradeFair + 'a, T: ?Sized + 'a> RwLockUpgradableReadGuard<'a, R, T> {
1972	/// Unlocks the `RwLock` using a fair unlock protocol.
1973	///
1974	/// By default, `RwLock` is unfair and allow the current thread to re-lock
1975	/// the `RwLock` before another has the chance to acquire the lock, even if
1976	/// that thread has been blocked on the `RwLock` for a long time. This is
1977	/// the default because it allows much higher throughput as it avoids
1978	/// forcing a context switch on every `RwLock` unlock. This can result in one
1979	/// thread acquiring a `RwLock` many more times than other threads.
1980	///
1981	/// However in some cases it can be beneficial to ensure fairness by forcing
1982	/// the lock to pass on to a waiting thread if there is one. This is done by
1983	/// using this method instead of dropping the `RwLockUpgradableReadGuard` normally.
1984	#[inline]
1985	pub fn unlock_fair(s: Self) {
1986	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
1987	unsafe {
1988	s.rwlock.raw.unlock_upgradable_fair();
1989	}
1990	mem::forget(s);
1991	}
1992
1993	/// Temporarily unlocks the `RwLock` to execute the given function.
1994	///
1995	/// The `RwLock` is unlocked a fair unlock protocol.
1996	///
1997	/// This is safe because `&mut` guarantees that there exist no other
1998	/// references to the data protected by the `RwLock`.
1999	#[inline]
2000	pub fn unlocked_fair<F, U>(s: &mut Self, f: F) -> U
2001	where
2002	F: FnOnce() -> U,
2003	{
2004	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2005	unsafe {
2006	s.rwlock.raw.unlock_upgradable_fair();
2007	}
2008	defer!(s.rwlock.raw.lock_upgradable());
2009	f()
2010	}
2011
2012	/// Temporarily yields the `RwLock` to a waiting thread if there is one.
2013	///
2014	/// This method is functionally equivalent to calling `unlock_fair` followed
2015	/// by `upgradable_read`, however it can be much more efficient in the case where there
2016	/// are no waiting threads.
2017	#[inline]
2018	pub fn bump(s: &mut Self) {
2019	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2020	unsafe {
2021	s.rwlock.raw.bump_upgradable();
2022	}
2023	}
2024	}
2025
2026	impl<'a, R: RawRwLockUpgradeDowngrade + 'a, T: ?Sized + 'a> RwLockUpgradableReadGuard<'a, R, T> {
2027	/// Atomically downgrades an upgradable read lock lock into a shared read lock
2028	/// without allowing any writers to take exclusive access of the lock in the
2029	/// meantime.
2030	///
2031	/// Note that if there are any writers currently waiting to take the lock
2032	/// then other readers may not be able to acquire the lock even if it was
2033	/// downgraded.
2034	pub fn downgrade(s: Self) -> RwLockReadGuard<'a, R, T> {
2035	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2036	unsafe {
2037	s.rwlock.raw.downgrade_upgradable();
2038	}
2039	let rwlock = s.rwlock;
2040	mem::forget(s);
2041	RwLockReadGuard {
2042	rwlock,
2043	marker: PhantomData,
2044	}
2045	}
2046
2047	/// First, atomically upgrades an upgradable read lock lock into an exclusive write lock,
2048	/// blocking the current thread until it can be acquired.
2049	///
2050	/// Then, calls the provided closure with an exclusive reference to the lock's data.
2051	///
2052	/// Finally, atomically downgrades the lock back to an upgradable read lock.
2053	/// The closure's return value is wrapped in `Some` and returned.
2054	///
2055	/// This function only requires a mutable reference to the guard, unlike
2056	/// `upgrade` which takes the guard by value.
2057	pub fn with_upgraded<Ret, F: FnOnce(&mut T) -> Ret>(&mut self, f: F) -> Ret {
2058	unsafe {
2059	self.rwlock.raw.upgrade();
2060	}
2061
2062	// Safety: We just upgraded the lock, so we have mutable access to the data.
2063	// This will restore the state the lock was in at the start of the function.
2064	defer!(unsafe { self.rwlock.raw.downgrade_to_upgradable() });
2065
2066	// Safety: We upgraded the lock, so we have mutable access to the data.
2067	// When this function returns, whether by drop or panic,
2068	// the drop guard will downgrade it back to an upgradeable lock.
2069	f(unsafe { &mut *self.rwlock.data.get() })
2070	}
2071
2072	/// First, tries to atomically upgrade an upgradable read lock into an exclusive write lock.
2073	///
2074	/// If the access could not be granted at this time, then `None` is returned.
2075	///
2076	/// Otherwise, calls the provided closure with an exclusive reference to the lock's data,
2077	/// and finally downgrades the lock back to an upgradable read lock.
2078	/// The closure's return value is wrapped in `Some` and returned.
2079	///
2080	/// This function only requires a mutable reference to the guard, unlike
2081	/// `try_upgrade` which takes the guard by value.
2082	pub fn try_with_upgraded<Ret, F: FnOnce(&mut T) -> Ret>(&mut self, f: F) -> Option<Ret> {
2083	if unsafe { self.rwlock.raw.try_upgrade() } {
2084	// Safety: We just upgraded the lock, so we have mutable access to the data.
2085	// This will restore the state the lock was in at the start of the function.
2086	defer!(unsafe { self.rwlock.raw.downgrade_to_upgradable() });
2087
2088	// Safety: We upgraded the lock, so we have mutable access to the data.
2089	// When this function returns, whether by drop or panic,
2090	// the drop guard will downgrade it back to an upgradeable lock.
2091	Some(f(unsafe { &mut *self.rwlock.data.get() }))
2092	} else {
2093	None
2094	}
2095	}
2096	}
2097
2098	impl<'a, R: RawRwLockUpgradeTimed + 'a, T: ?Sized + 'a> RwLockUpgradableReadGuard<'a, R, T> {
2099	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2100	/// write lock, until a timeout is reached.
2101	///
2102	/// If the access could not be granted before the timeout expires, then
2103	/// the current guard is returned.
2104	pub fn try_upgrade_for(
2105	s: Self,
2106	timeout: R::Duration,
2107	) -> Result<RwLockWriteGuard<'a, R, T>, Self> {
2108	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2109	if unsafe { s.rwlock.raw.try_upgrade_for(timeout) } {
2110	let rwlock = s.rwlock;
2111	mem::forget(s);
2112	Ok(RwLockWriteGuard {
2113	rwlock,
2114	marker: PhantomData,
2115	})
2116	} else {
2117	Err(s)
2118	}
2119	}
2120
2121	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2122	/// write lock, until a timeout is reached.
2123	///
2124	/// If the access could not be granted before the timeout expires, then
2125	/// the current guard is returned.
2126	#[inline]
2127	pub fn try_upgrade_until(
2128	s: Self,
2129	timeout: R::Instant,
2130	) -> Result<RwLockWriteGuard<'a, R, T>, Self> {
2131	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2132	if unsafe { s.rwlock.raw.try_upgrade_until(timeout) } {
2133	let rwlock = s.rwlock;
2134	mem::forget(s);
2135	Ok(RwLockWriteGuard {
2136	rwlock,
2137	marker: PhantomData,
2138	})
2139	} else {
2140	Err(s)
2141	}
2142	}
2143	}
2144
2145	impl<'a, R: RawRwLockUpgradeTimed + RawRwLockUpgradeDowngrade + 'a, T: ?Sized + 'a>
2146	RwLockUpgradableReadGuard<'a, R, T>
2147	{
2148	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2149	/// write lock, until a timeout is reached.
2150	///
2151	/// If the access could not be granted before the timeout expires, then
2152	/// `None` is returned.
2153	///
2154	/// Otherwise, calls the provided closure with an exclusive reference to the lock's data,
2155	/// and finally downgrades the lock back to an upgradable read lock.
2156	/// The closure's return value is wrapped in `Some` and returned.
2157	///
2158	/// This function only requires a mutable reference to the guard, unlike
2159	/// `try_upgrade_for` which takes the guard by value.
2160	pub fn try_with_upgraded_for<Ret, F: FnOnce(&mut T) -> Ret>(
2161	&mut self,
2162	timeout: R::Duration,
2163	f: F,
2164	) -> Option<Ret> {
2165	if unsafe { self.rwlock.raw.try_upgrade_for(timeout) } {
2166	// Safety: We just upgraded the lock, so we have mutable access to the data.
2167	// This will restore the state the lock was in at the start of the function.
2168	defer!(unsafe { self.rwlock.raw.downgrade_upgradable() });
2169
2170	// Safety: We upgraded the lock, so we have mutable access to the data.
2171	// When this function returns, whether by drop or panic,
2172	// the drop guard will downgrade it back to an upgradeable lock.
2173	Some(f(unsafe { &mut *self.rwlock.data.get() }))
2174	} else {
2175	None
2176	}
2177	}
2178
2179	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2180	/// write lock, until a timeout is reached.
2181	///
2182	/// If the access could not be granted before the timeout expires, then
2183	/// `None` is returned.
2184	///
2185	/// Otherwise, calls the provided closure with an exclusive reference to the lock's data,
2186	/// and finally downgrades the lock back to an upgradable read lock.
2187	/// The closure's return value is wrapped in `Some` and returned.
2188	///
2189	/// This function only requires a mutable reference to the guard, unlike
2190	/// `try_upgrade_until` which takes the guard by value.
2191	pub fn try_with_upgraded_until<Ret, F: FnOnce(&mut T) -> Ret>(
2192	&mut self,
2193	timeout: R::Instant,
2194	f: F,
2195	) -> Option<Ret> {
2196	if unsafe { self.rwlock.raw.try_upgrade_until(timeout) } {
2197	// Safety: We just upgraded the lock, so we have mutable access to the data.
2198	// This will restore the state the lock was in at the start of the function.
2199	defer!(unsafe { self.rwlock.raw.downgrade_upgradable() });
2200
2201	// Safety: We upgraded the lock, so we have mutable access to the data.
2202	// When this function returns, whether by drop or panic,
2203	// the drop guard will downgrade it back to an upgradeable lock.
2204	Some(f(unsafe { &mut *self.rwlock.data.get() }))
2205	} else {
2206	None
2207	}
2208	}
2209	}
2210
2211	impl<'a, R: RawRwLockUpgrade + 'a, T: ?Sized + 'a> Deref for RwLockUpgradableReadGuard<'a, R, T> {
2212	type Target = T;
2213	#[inline]
2214	fn deref(&self) -> &T {
2215	unsafe { &*self.rwlock.data.get() }
2216	}
2217	}
2218
2219	impl<'a, R: RawRwLockUpgrade + 'a, T: ?Sized + 'a> Drop for RwLockUpgradableReadGuard<'a, R, T> {
2220	#[inline]
2221	fn drop(&mut self) {
2222	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2223	unsafe {
2224	self.rwlock.raw.unlock_upgradable();
2225	}
2226	}
2227	}
2228
2229	impl<'a, R: RawRwLockUpgrade + 'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug
2230	for RwLockUpgradableReadGuard<'a, R, T>
2231	{
2232	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2233	fmt::Debug::fmt(&**self, f)
2234	}
2235	}
2236
2237	impl<'a, R: RawRwLockUpgrade + 'a, T: fmt::Display + ?Sized + 'a> fmt::Display
2238	for RwLockUpgradableReadGuard<'a, R, T>
2239	{
2240	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2241	(**self).fmt(f)
2242	}
2243	}
2244
2245	#[cfg(feature = "owning_ref")]
2246	unsafe impl<'a, R: RawRwLockUpgrade + 'a, T: ?Sized + 'a> StableAddress
2247	for RwLockUpgradableReadGuard<'a, R, T>
2248	{
2249	}
2250
2251	/// An RAII rwlock guard returned by the `Arc` locking operations on `RwLock`.
2252	/// This is similar to the `RwLockUpgradableReadGuard` struct, except instead of using a reference to unlock the
2253	/// `RwLock` it uses an `Arc<RwLock>`. This has several advantages, most notably that it has an `'static`
2254	/// lifetime.
2255	#[cfg(feature = "arc_lock")]
2256	#[clippy::has_significant_drop]
2257	#[must_use = "if unused the RwLock will immediately unlock"]
2258	pub struct ArcRwLockUpgradableReadGuard<R: RawRwLockUpgrade, T: ?Sized> {
2259	rwlock: Arc<RwLock<R, T>>,
2260	marker: PhantomData<R::GuardMarker>,
2261	}
2262
2263	#[cfg(feature = "arc_lock")]
2264	impl<R: RawRwLockUpgrade, T: ?Sized> ArcRwLockUpgradableReadGuard<R, T> {
2265	/// Returns a reference to the rwlock, contained in its original `Arc`.
2266	pub fn rwlock(s: &Self) -> &Arc<RwLock<R, T>> {
2267	&s.rwlock
2268	}
2269
2270	/// Temporarily unlocks the `RwLock` to execute the given function.
2271	///
2272	/// This is functionally identical to the `unlocked` method on [`RwLockUpgradableReadGuard`].
2273	#[inline]
2274	pub fn unlocked<F, U>(s: &mut Self, f: F) -> U
2275	where
2276	F: FnOnce() -> U,
2277	{
2278	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2279	unsafe {
2280	s.rwlock.raw.unlock_upgradable();
2281	}
2282	defer!(s.rwlock.raw.lock_upgradable());
2283	f()
2284	}
2285
2286	/// Atomically upgrades an upgradable read lock lock into an exclusive write lock,
2287	/// blocking the current thread until it can be acquired.
2288	pub fn upgrade(s: Self) -> ArcRwLockWriteGuard<R, T> {
2289	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2290	unsafe {
2291	s.rwlock.raw.upgrade();
2292	}
2293
2294	// SAFETY: avoid incrementing or decrementing the refcount using ManuallyDrop and reading the Arc out
2295	// of the struct
2296	let s = ManuallyDrop::new(s);
2297	let rwlock = unsafe { ptr::read(&s.rwlock) };
2298
2299	ArcRwLockWriteGuard {
2300	rwlock,
2301	marker: PhantomData,
2302	}
2303	}
2304
2305	/// Tries to atomically upgrade an upgradable read lock into an exclusive write lock.
2306	///
2307	/// If the access could not be granted at this time, then the current guard is returned.
2308	pub fn try_upgrade(s: Self) -> Result<ArcRwLockWriteGuard<R, T>, Self> {
2309	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2310	if unsafe { s.rwlock.raw.try_upgrade() } {
2311	// SAFETY: same as above
2312	let s = ManuallyDrop::new(s);
2313	let rwlock = unsafe { ptr::read(&s.rwlock) };
2314
2315	Ok(ArcRwLockWriteGuard {
2316	rwlock,
2317	marker: PhantomData,
2318	})
2319	} else {
2320	Err(s)
2321	}
2322	}
2323	}
2324
2325	#[cfg(feature = "arc_lock")]
2326	impl<R: RawRwLockUpgradeFair, T: ?Sized> ArcRwLockUpgradableReadGuard<R, T> {
2327	/// Unlocks the `RwLock` using a fair unlock protocol.
2328	///
2329	/// This is functionally identical to the `unlock_fair` method on [`RwLockUpgradableReadGuard`].
2330	#[inline]
2331	pub fn unlock_fair(s: Self) {
2332	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2333	unsafe {
2334	s.rwlock.raw.unlock_upgradable_fair();
2335	}
2336
2337	// SAFETY: make sure we decrement the refcount properly
2338	let mut s = ManuallyDrop::new(s);
2339	unsafe { ptr::drop_in_place(&mut s.rwlock) };
2340	}
2341
2342	/// Temporarily unlocks the `RwLock` to execute the given function.
2343	///
2344	/// This is functionally equivalent to the `unlocked_fair` method on [`RwLockUpgradableReadGuard`].
2345	#[inline]
2346	pub fn unlocked_fair<F, U>(s: &mut Self, f: F) -> U
2347	where
2348	F: FnOnce() -> U,
2349	{
2350	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2351	unsafe {
2352	s.rwlock.raw.unlock_upgradable_fair();
2353	}
2354	defer!(s.rwlock.raw.lock_upgradable());
2355	f()
2356	}
2357
2358	/// Temporarily yields the `RwLock` to a waiting thread if there is one.
2359	///
2360	/// This method is functionally equivalent to calling `bump` on [`RwLockUpgradableReadGuard`].
2361	#[inline]
2362	pub fn bump(s: &mut Self) {
2363	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2364	unsafe {
2365	s.rwlock.raw.bump_upgradable();
2366	}
2367	}
2368	}
2369
2370	#[cfg(feature = "arc_lock")]
2371	impl<R: RawRwLockUpgradeDowngrade, T: ?Sized> ArcRwLockUpgradableReadGuard<R, T> {
2372	/// Atomically downgrades an upgradable read lock lock into a shared read lock
2373	/// without allowing any writers to take exclusive access of the lock in the
2374	/// meantime.
2375	///
2376	/// Note that if there are any writers currently waiting to take the lock
2377	/// then other readers may not be able to acquire the lock even if it was
2378	/// downgraded.
2379	pub fn downgrade(s: Self) -> ArcRwLockReadGuard<R, T> {
2380	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2381	unsafe {
2382	s.rwlock.raw.downgrade_upgradable();
2383	}
2384
2385	// SAFETY: use ManuallyDrop and ptr::read to ensure the refcount is not changed
2386	let s = ManuallyDrop::new(s);
2387	let rwlock = unsafe { ptr::read(&s.rwlock) };
2388
2389	ArcRwLockReadGuard {
2390	rwlock,
2391	marker: PhantomData,
2392	}
2393	}
2394
2395	/// First, atomically upgrades an upgradable read lock lock into an exclusive write lock,
2396	/// blocking the current thread until it can be acquired.
2397	///
2398	/// Then, calls the provided closure with an exclusive reference to the lock's data.
2399	///
2400	/// Finally, atomically downgrades the lock back to an upgradable read lock.
2401	/// The closure's return value is returned.
2402	///
2403	/// This function only requires a mutable reference to the guard, unlike
2404	/// `upgrade` which takes the guard by value.
2405	pub fn with_upgraded<Ret, F: FnOnce(&mut T) -> Ret>(&mut self, f: F) -> Ret {
2406	unsafe {
2407	self.rwlock.raw.upgrade();
2408	}
2409
2410	// Safety: We just upgraded the lock, so we have mutable access to the data.
2411	// This will restore the state the lock was in at the start of the function.
2412	defer!(unsafe { self.rwlock.raw.downgrade_upgradable() });
2413
2414	// Safety: We upgraded the lock, so we have mutable access to the data.
2415	// When this function returns, whether by drop or panic,
2416	// the drop guard will downgrade it back to an upgradeable lock.
2417	f(unsafe { &mut *self.rwlock.data.get() })
2418	}
2419
2420	/// First, tries to atomically upgrade an upgradable read lock into an exclusive write lock.
2421	///
2422	/// If the access could not be granted at this time, then `None` is returned.
2423	///
2424	/// Otherwise, calls the provided closure with an exclusive reference to the lock's data,
2425	/// and finally downgrades the lock back to an upgradable read lock.
2426	/// The closure's return value is wrapped in `Some` and returned.
2427	///
2428	/// This function only requires a mutable reference to the guard, unlike
2429	/// `try_upgrade` which takes the guard by value.
2430	pub fn try_with_upgraded<Ret, F: FnOnce(&mut T) -> Ret>(&mut self, f: F) -> Option<Ret> {
2431	if unsafe { self.rwlock.raw.try_upgrade() } {
2432	// Safety: We just upgraded the lock, so we have mutable access to the data.
2433	// This will restore the state the lock was in at the start of the function.
2434	defer!(unsafe { self.rwlock.raw.downgrade_upgradable() });
2435
2436	// Safety: We upgraded the lock, so we have mutable access to the data.
2437	// When this function returns, whether by drop or panic,
2438	// the drop guard will downgrade it back to an upgradeable lock.
2439	Some(f(unsafe { &mut *self.rwlock.data.get() }))
2440	} else {
2441	None
2442	}
2443	}
2444	}
2445
2446	#[cfg(feature = "arc_lock")]
2447	impl<R: RawRwLockUpgradeTimed, T: ?Sized> ArcRwLockUpgradableReadGuard<R, T> {
2448	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2449	/// write lock, until a timeout is reached.
2450	///
2451	/// If the access could not be granted before the timeout expires, then
2452	/// the current guard is returned.
2453	pub fn try_upgrade_for(
2454	s: Self,
2455	timeout: R::Duration,
2456	) -> Result<ArcRwLockWriteGuard<R, T>, Self> {
2457	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2458	if unsafe { s.rwlock.raw.try_upgrade_for(timeout) } {
2459	// SAFETY: same as above
2460	let s = ManuallyDrop::new(s);
2461	let rwlock = unsafe { ptr::read(&s.rwlock) };
2462
2463	Ok(ArcRwLockWriteGuard {
2464	rwlock,
2465	marker: PhantomData,
2466	})
2467	} else {
2468	Err(s)
2469	}
2470	}
2471
2472	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2473	/// write lock, until a timeout is reached.
2474	///
2475	/// If the access could not be granted before the timeout expires, then
2476	/// the current guard is returned.
2477	#[inline]
2478	pub fn try_upgrade_until(
2479	s: Self,
2480	timeout: R::Instant,
2481	) -> Result<ArcRwLockWriteGuard<R, T>, Self> {
2482	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2483	if unsafe { s.rwlock.raw.try_upgrade_until(timeout) } {
2484	// SAFETY: same as above
2485	let s = ManuallyDrop::new(s);
2486	let rwlock = unsafe { ptr::read(&s.rwlock) };
2487
2488	Ok(ArcRwLockWriteGuard {
2489	rwlock,
2490	marker: PhantomData,
2491	})
2492	} else {
2493	Err(s)
2494	}
2495	}
2496	}
2497
2498	#[cfg(feature = "arc_lock")]
2499	impl<R: RawRwLockUpgradeTimed + RawRwLockUpgradeDowngrade, T: ?Sized>
2500	ArcRwLockUpgradableReadGuard<R, T>
2501	{
2502	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2503	/// write lock, until a timeout is reached.
2504	///
2505	/// If the access could not be granted before the timeout expires, then
2506	/// `None` is returned.
2507	///
2508	/// Otherwise, calls the provided closure with an exclusive reference to the lock's data,
2509	/// and finally downgrades the lock back to an upgradable read lock.
2510	/// The closure's return value is wrapped in `Some` and returned.
2511	///
2512	/// This function only requires a mutable reference to the guard, unlike
2513	/// `try_upgrade_for` which takes the guard by value.
2514	pub fn try_with_upgraded_for<Ret, F: FnOnce(&mut T) -> Ret>(
2515	&mut self,
2516	timeout: R::Duration,
2517	f: F,
2518	) -> Option<Ret> {
2519	if unsafe { self.rwlock.raw.try_upgrade_for(timeout) } {
2520	// Safety: We just upgraded the lock, so we have mutable access to the data.
2521	// This will restore the state the lock was in at the start of the function.
2522	defer!(unsafe { self.rwlock.raw.downgrade_upgradable() });
2523
2524	// Safety: We upgraded the lock, so we have mutable access to the data.
2525	// When this function returns, whether by drop or panic,
2526	// the drop guard will downgrade it back to an upgradeable lock.
2527	Some(f(unsafe { &mut *self.rwlock.data.get() }))
2528	} else {
2529	None
2530	}
2531	}
2532
2533	/// Tries to atomically upgrade an upgradable read lock into an exclusive
2534	/// write lock, until a timeout is reached.
2535	///
2536	/// If the access could not be granted before the timeout expires, then
2537	/// `None` is returned.
2538	///
2539	/// Otherwise, calls the provided closure with an exclusive reference to the lock's data,
2540	/// and finally downgrades the lock back to an upgradable read lock.
2541	/// The closure's return value is wrapped in `Some` and returned.
2542	///
2543	/// This function only requires a mutable reference to the guard, unlike
2544	/// `try_upgrade_until` which takes the guard by value.
2545	pub fn try_with_upgraded_until<Ret, F: FnOnce(&mut T) -> Ret>(
2546	&mut self,
2547	timeout: R::Instant,
2548	f: F,
2549	) -> Option<Ret> {
2550	if unsafe { self.rwlock.raw.try_upgrade_until(timeout) } {
2551	// Safety: We just upgraded the lock, so we have mutable access to the data.
2552	// This will restore the state the lock was in at the start of the function.
2553	defer!(unsafe { self.rwlock.raw.downgrade_upgradable() });
2554
2555	// Safety: We upgraded the lock, so we have mutable access to the data.
2556	// When this function returns, whether by drop or panic,
2557	// the drop guard will downgrade it back to an upgradeable lock.
2558	Some(f(unsafe { &mut *self.rwlock.data.get() }))
2559	} else {
2560	None
2561	}
2562	}
2563	}
2564
2565	#[cfg(feature = "arc_lock")]
2566	impl<R: RawRwLockUpgrade, T: ?Sized> Deref for ArcRwLockUpgradableReadGuard<R, T> {
2567	type Target = T;
2568	#[inline]
2569	fn deref(&self) -> &T {
2570	unsafe { &*self.rwlock.data.get() }
2571	}
2572	}
2573
2574	#[cfg(feature = "arc_lock")]
2575	impl<R: RawRwLockUpgrade, T: ?Sized> Drop for ArcRwLockUpgradableReadGuard<R, T> {
2576	#[inline]
2577	fn drop(&mut self) {
2578	// Safety: An RwLockUpgradableReadGuard always holds an upgradable lock.
2579	unsafe {
2580	self.rwlock.raw.unlock_upgradable();
2581	}
2582	}
2583	}
2584
2585	#[cfg(feature = "arc_lock")]
2586	impl<R: RawRwLockUpgrade, T: fmt::Debug + ?Sized> fmt::Debug
2587	for ArcRwLockUpgradableReadGuard<R, T>
2588	{
2589	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2590	fmt::Debug::fmt(&**self, f)
2591	}
2592	}
2593
2594	#[cfg(feature = "arc_lock")]
2595	impl<R: RawRwLockUpgrade, T: fmt::Display + ?Sized> fmt::Display
2596	for ArcRwLockUpgradableReadGuard<R, T>
2597	{
2598	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2599	(**self).fmt(f)
2600	}
2601	}
2602
2603	/// An RAII read lock guard returned by `RwLockReadGuard::map`, which can point to a
2604	/// subfield of the protected data.
2605	///
2606	/// The main difference between `MappedRwLockReadGuard` and `RwLockReadGuard` is that the
2607	/// former doesn't support temporarily unlocking and re-locking, since that
2608	/// could introduce soundness issues if the locked object is modified by another
2609	/// thread.
2610	#[clippy::has_significant_drop]
2611	#[must_use = "if unused the RwLock will immediately unlock"]
2612	pub struct MappedRwLockReadGuard<'a, R: RawRwLock, T: ?Sized> {
2613	raw: &'a R,
2614	data: *const T,
2615	marker: PhantomData<&'a T>,
2616	}
2617
2618	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + Sync + 'a> Sync for MappedRwLockReadGuard<'a, R, T> {}
2619	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + Sync + 'a> Send for MappedRwLockReadGuard<'a, R, T> where
2620	R::GuardMarker: Send
2621	{
2622	}
2623
2624	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> MappedRwLockReadGuard<'a, R, T> {
2625	/// Make a new `MappedRwLockReadGuard` for a component of the locked data.
2626	///
2627	/// This operation cannot fail as the `MappedRwLockReadGuard` passed
2628	/// in already locked the data.
2629	///
2630	/// This is an associated function that needs to be
2631	/// used as `MappedRwLockReadGuard::map(...)`. A method would interfere with methods of
2632	/// the same name on the contents of the locked data.
2633	#[inline]
2634	pub fn map<U: ?Sized, F>(s: Self, f: F) -> MappedRwLockReadGuard<'a, R, U>
2635	where
2636	F: FnOnce(&T) -> &U,
2637	{
2638	let raw = s.raw;
2639	let data = f(unsafe { &*s.data });
2640	mem::forget(s);
2641	MappedRwLockReadGuard {
2642	raw,
2643	data,
2644	marker: PhantomData,
2645	}
2646	}
2647
2648	/// Attempts to make a new `MappedRwLockReadGuard` for a component of the
2649	/// locked data. The original guard is return if the closure returns `None`.
2650	///
2651	/// This operation cannot fail as the `MappedRwLockReadGuard` passed
2652	/// in already locked the data.
2653	///
2654	/// This is an associated function that needs to be
2655	/// used as `MappedRwLockReadGuard::try_map(...)`. A method would interfere with methods of
2656	/// the same name on the contents of the locked data.
2657	#[inline]
2658	pub fn try_map<U: ?Sized, F>(s: Self, f: F) -> Result<MappedRwLockReadGuard<'a, R, U>, Self>
2659	where
2660	F: FnOnce(&T) -> Option<&U>,
2661	{
2662	let raw = s.raw;
2663	let data = match f(unsafe { &*s.data }) {
2664	Some(data) => data,
2665	None => return Err(s),
2666	};
2667	mem::forget(s);
2668	Ok(MappedRwLockReadGuard {
2669	raw,
2670	data,
2671	marker: PhantomData,
2672	})
2673	}
2674	}
2675
2676	impl<'a, R: RawRwLockFair + 'a, T: ?Sized + 'a> MappedRwLockReadGuard<'a, R, T> {
2677	/// Unlocks the `RwLock` using a fair unlock protocol.
2678	///
2679	/// By default, `RwLock` is unfair and allow the current thread to re-lock
2680	/// the `RwLock` before another has the chance to acquire the lock, even if
2681	/// that thread has been blocked on the `RwLock` for a long time. This is
2682	/// the default because it allows much higher throughput as it avoids
2683	/// forcing a context switch on every `RwLock` unlock. This can result in one
2684	/// thread acquiring a `RwLock` many more times than other threads.
2685	///
2686	/// However in some cases it can be beneficial to ensure fairness by forcing
2687	/// the lock to pass on to a waiting thread if there is one. This is done by
2688	/// using this method instead of dropping the `MappedRwLockReadGuard` normally.
2689	#[inline]
2690	pub fn unlock_fair(s: Self) {
2691	// Safety: A MappedRwLockReadGuard always holds a shared lock.
2692	unsafe {
2693	s.raw.unlock_shared_fair();
2694	}
2695	mem::forget(s);
2696	}
2697	}
2698
2699	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Deref for MappedRwLockReadGuard<'a, R, T> {
2700	type Target = T;
2701	#[inline]
2702	fn deref(&self) -> &T {
2703	unsafe { &*self.data }
2704	}
2705	}
2706
2707	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Drop for MappedRwLockReadGuard<'a, R, T> {
2708	#[inline]
2709	fn drop(&mut self) {
2710	// Safety: A MappedRwLockReadGuard always holds a shared lock.
2711	unsafe {
2712	self.raw.unlock_shared();
2713	}
2714	}
2715	}
2716
2717	impl<'a, R: RawRwLock + 'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug
2718	for MappedRwLockReadGuard<'a, R, T>
2719	{
2720	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2721	fmt::Debug::fmt(&**self, f)
2722	}
2723	}
2724
2725	impl<'a, R: RawRwLock + 'a, T: fmt::Display + ?Sized + 'a> fmt::Display
2726	for MappedRwLockReadGuard<'a, R, T>
2727	{
2728	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2729	(**self).fmt(f)
2730	}
2731	}
2732
2733	#[cfg(feature = "owning_ref")]
2734	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> StableAddress
2735	for MappedRwLockReadGuard<'a, R, T>
2736	{
2737	}
2738
2739	/// An RAII write lock guard returned by `RwLockWriteGuard::map`, which can point to a
2740	/// subfield of the protected data.
2741	///
2742	/// The main difference between `MappedRwLockWriteGuard` and `RwLockWriteGuard` is that the
2743	/// former doesn't support temporarily unlocking and re-locking, since that
2744	/// could introduce soundness issues if the locked object is modified by another
2745	/// thread.
2746	#[clippy::has_significant_drop]
2747	#[must_use = "if unused the RwLock will immediately unlock"]
2748	pub struct MappedRwLockWriteGuard<'a, R: RawRwLock, T: ?Sized> {
2749	raw: &'a R,
2750	data: *mut T,
2751	marker: PhantomData<&'a mut T>,
2752	}
2753
2754	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + Sync + 'a> Sync
2755	for MappedRwLockWriteGuard<'a, R, T>
2756	{
2757	}
2758	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + Send + 'a> Send for MappedRwLockWriteGuard<'a, R, T> where
2759	R::GuardMarker: Send
2760	{
2761	}
2762
2763	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> MappedRwLockWriteGuard<'a, R, T> {
2764	/// Make a new `MappedRwLockWriteGuard` for a component of the locked data.
2765	///
2766	/// This operation cannot fail as the `MappedRwLockWriteGuard` passed
2767	/// in already locked the data.
2768	///
2769	/// This is an associated function that needs to be
2770	/// used as `MappedRwLockWriteGuard::map(...)`. A method would interfere with methods of
2771	/// the same name on the contents of the locked data.
2772	#[inline]
2773	pub fn map<U: ?Sized, F>(s: Self, f: F) -> MappedRwLockWriteGuard<'a, R, U>
2774	where
2775	F: FnOnce(&mut T) -> &mut U,
2776	{
2777	let raw = s.raw;
2778	let data = f(unsafe { &mut *s.data });
2779	mem::forget(s);
2780	MappedRwLockWriteGuard {
2781	raw,
2782	data,
2783	marker: PhantomData,
2784	}
2785	}
2786
2787	/// Attempts to make a new `MappedRwLockWriteGuard` for a component of the
2788	/// locked data. The original guard is return if the closure returns `None`.
2789	///
2790	/// This operation cannot fail as the `MappedRwLockWriteGuard` passed
2791	/// in already locked the data.
2792	///
2793	/// This is an associated function that needs to be
2794	/// used as `MappedRwLockWriteGuard::try_map(...)`. A method would interfere with methods of
2795	/// the same name on the contents of the locked data.
2796	#[inline]
2797	pub fn try_map<U: ?Sized, F>(s: Self, f: F) -> Result<MappedRwLockWriteGuard<'a, R, U>, Self>
2798	where
2799	F: FnOnce(&mut T) -> Option<&mut U>,
2800	{
2801	let raw = s.raw;
2802	let data = match f(unsafe { &mut *s.data }) {
2803	Some(data) => data,
2804	None => return Err(s),
2805	};
2806	mem::forget(s);
2807	Ok(MappedRwLockWriteGuard {
2808	raw,
2809	data,
2810	marker: PhantomData,
2811	})
2812	}
2813	}
2814
2815	impl<'a, R: RawRwLockFair + 'a, T: ?Sized + 'a> MappedRwLockWriteGuard<'a, R, T> {
2816	/// Unlocks the `RwLock` using a fair unlock protocol.
2817	///
2818	/// By default, `RwLock` is unfair and allow the current thread to re-lock
2819	/// the `RwLock` before another has the chance to acquire the lock, even if
2820	/// that thread has been blocked on the `RwLock` for a long time. This is
2821	/// the default because it allows much higher throughput as it avoids
2822	/// forcing a context switch on every `RwLock` unlock. This can result in one
2823	/// thread acquiring a `RwLock` many more times than other threads.
2824	///
2825	/// However in some cases it can be beneficial to ensure fairness by forcing
2826	/// the lock to pass on to a waiting thread if there is one. This is done by
2827	/// using this method instead of dropping the `MappedRwLockWriteGuard` normally.
2828	#[inline]
2829	pub fn unlock_fair(s: Self) {
2830	// Safety: A MappedRwLockWriteGuard always holds an exclusive lock.
2831	unsafe {
2832	s.raw.unlock_exclusive_fair();
2833	}
2834	mem::forget(s);
2835	}
2836	}
2837
2838	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Deref for MappedRwLockWriteGuard<'a, R, T> {
2839	type Target = T;
2840	#[inline]
2841	fn deref(&self) -> &T {
2842	unsafe { &*self.data }
2843	}
2844	}
2845
2846	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> DerefMut for MappedRwLockWriteGuard<'a, R, T> {
2847	#[inline]
2848	fn deref_mut(&mut self) -> &mut T {
2849	unsafe { &mut *self.data }
2850	}
2851	}
2852
2853	impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> Drop for MappedRwLockWriteGuard<'a, R, T> {
2854	#[inline]
2855	fn drop(&mut self) {
2856	// Safety: A MappedRwLockWriteGuard always holds an exclusive lock.
2857	unsafe {
2858	self.raw.unlock_exclusive();
2859	}
2860	}
2861	}
2862
2863	impl<'a, R: RawRwLock + 'a, T: fmt::Debug + ?Sized + 'a> fmt::Debug
2864	for MappedRwLockWriteGuard<'a, R, T>
2865	{
2866	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2867	fmt::Debug::fmt(&**self, f)
2868	}
2869	}
2870
2871	impl<'a, R: RawRwLock + 'a, T: fmt::Display + ?Sized + 'a> fmt::Display
2872	for MappedRwLockWriteGuard<'a, R, T>
2873	{
2874	fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2875	(**self).fmt(f)
2876	}
2877	}
2878
2879	#[cfg(feature = "owning_ref")]
2880	unsafe impl<'a, R: RawRwLock + 'a, T: ?Sized + 'a> StableAddress
2881	for MappedRwLockWriteGuard<'a, R, T>
2882	{
2883	}
2884