1use super::lazy::LazyKeyInner;
2use crate::cell::Cell;
3use crate::sys::thread_local_dtor::register_dtor;
4use crate::{fmt, mem, panic};
5
6#[doc(hidden)]
7#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)]
8#[allow_internal_unsafe]
9#[unstable(feature = "thread_local_internals", issue = "none")]
10#[rustc_macro_transparency = "semitransparent"]
11pub macro thread_local_inner {
12 // used to generate the `LocalKey` value for const-initialized thread locals
13 (@key $t:ty, const $init:expr) => {{
14 #[inline]
15 #[deny(unsafe_op_in_unsafe_fn)]
16 // FIXME: Use `SyncUnsafeCell` instead of allowing `static_mut_refs` lint
17 #[cfg_attr(not(bootstrap), allow(static_mut_refs))]
18 unsafe fn __getit(
19 _init: $crate::option::Option<&mut $crate::option::Option<$t>>,
20 ) -> $crate::option::Option<&'static $t> {
21 const INIT_EXPR: $t = $init;
22 // If the platform has support for `#[thread_local]`, use it.
23 #[thread_local]
24 static mut VAL: $t = INIT_EXPR;
25
26 // If a dtor isn't needed we can do something "very raw" and
27 // just get going.
28 if !$crate::mem::needs_drop::<$t>() {
29 unsafe {
30 return $crate::option::Option::Some(&VAL)
31 }
32 }
33
34 // 0 == dtor not registered
35 // 1 == dtor registered, dtor not run
36 // 2 == dtor registered and is running or has run
37 #[thread_local]
38 static STATE: $crate::cell::Cell<$crate::primitive::u8> = $crate::cell::Cell::new(0);
39
40 // Safety: Performs `drop_in_place(ptr as *mut $t)`, and requires
41 // all that comes with it.
42 unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) {
43 $crate::thread::local_impl::abort_on_dtor_unwind(|| {
44 let old_state = STATE.replace(2);
45 $crate::debug_assert_eq!(old_state, 1);
46 // Safety: safety requirement is passed on to caller.
47 unsafe { $crate::ptr::drop_in_place(ptr.cast::<$t>()); }
48 });
49 }
50
51 unsafe {
52 match STATE.get() {
53 // 0 == we haven't registered a destructor, so do
54 // so now.
55 0 => {
56 $crate::thread::local_impl::Key::<$t>::register_dtor(
57 $crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8,
58 destroy,
59 );
60 STATE.set(1);
61 $crate::option::Option::Some(&VAL)
62 }
63 // 1 == the destructor is registered and the value
64 // is valid, so return the pointer.
65 1 => $crate::option::Option::Some(&VAL),
66 // otherwise the destructor has already run, so we
67 // can't give access.
68 _ => $crate::option::Option::None,
69 }
70 }
71 }
72
73 unsafe {
74 $crate::thread::LocalKey::new(__getit)
75 }
76 }},
77
78 // used to generate the `LocalKey` value for `thread_local!`
79 (@key $t:ty, $init:expr) => {
80 {
81 #[inline]
82 fn __init() -> $t { $init }
83
84 #[inline]
85 unsafe fn __getit(
86 init: $crate::option::Option<&mut $crate::option::Option<$t>>,
87 ) -> $crate::option::Option<&'static $t> {
88 #[thread_local]
89 static __KEY: $crate::thread::local_impl::Key<$t> =
90 $crate::thread::local_impl::Key::<$t>::new();
91
92 unsafe {
93 __KEY.get(move || {
94 if let $crate::option::Option::Some(init) = init {
95 if let $crate::option::Option::Some(value) = init.take() {
96 return value;
97 } else if $crate::cfg!(debug_assertions) {
98 $crate::unreachable!("missing default value");
99 }
100 }
101 __init()
102 })
103 }
104 }
105
106 unsafe {
107 $crate::thread::LocalKey::new(__getit)
108 }
109 }
110 },
111 ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
112 $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
113 $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*);
114 },
115}
116
117#[derive(Copy, Clone)]
118enum DtorState {
119 Unregistered,
120 Registered,
121 RunningOrHasRun,
122}
123
124// This data structure has been carefully constructed so that the fast path
125// only contains one branch on x86. That optimization is necessary to avoid
126// duplicated tls lookups on OSX.
127//
128// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
129pub struct Key<T> {
130 // If `LazyKeyInner::get` returns `None`, that indicates either:
131 // * The value has never been initialized
132 // * The value is being recursively initialized
133 // * The value has already been destroyed or is being destroyed
134 // To determine which kind of `None`, check `dtor_state`.
135 //
136 // This is very optimizer friendly for the fast path - initialized but
137 // not yet dropped.
138 inner: LazyKeyInner<T>,
139
140 // Metadata to keep track of the state of the destructor. Remember that
141 // this variable is thread-local, not global.
142 dtor_state: Cell<DtorState>,
143}
144
145impl<T> fmt::Debug for Key<T> {
146 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
147 f.debug_struct(name:"Key").finish_non_exhaustive()
148 }
149}
150impl<T> Key<T> {
151 pub const fn new() -> Key<T> {
152 Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
153 }
154
155 // note that this is just a publicly-callable function only for the
156 // const-initialized form of thread locals, basically a way to call the
157 // free `register_dtor` function defined elsewhere in std.
158 pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
159 unsafe {
160 register_dtor(a, dtor);
161 }
162 }
163
164 pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
165 // SAFETY: See the definitions of `LazyKeyInner::get` and
166 // `try_initialize` for more information.
167 //
168 // The caller must ensure no mutable references are ever active to
169 // the inner cell or the inner T when this is called.
170 // The `try_initialize` is dependant on the passed `init` function
171 // for this.
172 unsafe {
173 match self.inner.get() {
174 Some(val) => Some(val),
175 None => self.try_initialize(init),
176 }
177 }
178 }
179
180 // `try_initialize` is only called once per fast thread local variable,
181 // except in corner cases where thread_local dtors reference other
182 // thread_local's, or it is being recursively initialized.
183 //
184 // Macos: Inlining this function can cause two `tlv_get_addr` calls to
185 // be performed for every call to `Key::get`.
186 // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
187 #[inline(never)]
188 unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
189 // SAFETY: See comment above (this function doc).
190 if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } {
191 // SAFETY: See comment above (this function doc).
192 Some(unsafe { self.inner.initialize(init) })
193 } else {
194 None
195 }
196 }
197
198 // `try_register_dtor` is only called once per fast thread local
199 // variable, except in corner cases where thread_local dtors reference
200 // other thread_local's, or it is being recursively initialized.
201 unsafe fn try_register_dtor(&self) -> bool {
202 match self.dtor_state.get() {
203 DtorState::Unregistered => {
204 // SAFETY: dtor registration happens before initialization.
205 // Passing `self` as a pointer while using `destroy_value<T>`
206 // is safe because the function will build a pointer to a
207 // Key<T>, which is the type of self and so find the correct
208 // size.
209 unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
210 self.dtor_state.set(DtorState::Registered);
211 true
212 }
213 DtorState::Registered => {
214 // recursively initialized
215 true
216 }
217 DtorState::RunningOrHasRun => false,
218 }
219 }
220}
221
222unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
223 let ptr: *mut Key = ptr as *mut Key<T>;
224
225 // SAFETY:
226 //
227 // The pointer `ptr` has been built just above and comes from
228 // `try_register_dtor` where it is originally a Key<T> coming from `self`,
229 // making it non-NUL and of the correct type.
230 //
231 // Right before we run the user destructor be sure to set the
232 // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
233 // causes future calls to `get` to run `try_initialize_drop` again,
234 // which will now fail, and return `None`.
235 //
236 // Wrap the call in a catch to ensure unwinding is caught in the event
237 // a panic takes place in a destructor.
238 if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe {
239 let value: Option = (*ptr).inner.take();
240 (*ptr).dtor_state.set(val:DtorState::RunningOrHasRun);
241 drop(value);
242 })) {
243 rtabort!("thread local panicked on drop");
244 }
245}
246