1use super::lazy::LazyKeyInner;
2use crate::cell::Cell;
3use crate::sys::thread_local_dtor::register_dtor;
4use crate::{fmt, mem, panic};
5
6#[doc(hidden)]
7#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)]
8#[allow_internal_unsafe]
9#[unstable(feature = "thread_local_internals", issue = "none")]
10#[rustc_macro_transparency = "semitransparent"]
11pub macro thread_local_inner {
12 // used to generate the `LocalKey` value for const-initialized thread locals
13 (@key $t:ty, const $init:expr) => {{
14 #[inline]
15 #[deny(unsafe_op_in_unsafe_fn)]
16 unsafe fn __getit(
17 _init: $crate::option::Option<&mut $crate::option::Option<$t>>,
18 ) -> $crate::option::Option<&'static $t> {
19 const INIT_EXPR: $t = $init;
20 // If the platform has support for `#[thread_local]`, use it.
21 #[thread_local]
22 // We use `UnsafeCell` here instead of `static mut` to ensure any generated TLS shims
23 // have a nonnull attribute on their return value.
24 static VAL: $crate::cell::UnsafeCell<$t> = $crate::cell::UnsafeCell::new(INIT_EXPR);
25
26 // If a dtor isn't needed we can do something "very raw" and
27 // just get going.
28 if !$crate::mem::needs_drop::<$t>() {
29 unsafe {
30 return $crate::option::Option::Some(&*VAL.get())
31 }
32 }
33
34 // 0 == dtor not registered
35 // 1 == dtor registered, dtor not run
36 // 2 == dtor registered and is running or has run
37 #[thread_local]
38 static STATE: $crate::cell::Cell<$crate::primitive::u8> = $crate::cell::Cell::new(0);
39
40 // Safety: Performs `drop_in_place(ptr as *mut $t)`, and requires
41 // all that comes with it.
42 unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) {
43 $crate::thread::local_impl::abort_on_dtor_unwind(|| {
44 let old_state = STATE.replace(2);
45 $crate::debug_assert_eq!(old_state, 1);
46 // Safety: safety requirement is passed on to caller.
47 unsafe { $crate::ptr::drop_in_place(ptr.cast::<$t>()); }
48 });
49 }
50
51 unsafe {
52 match STATE.get() {
53 // 0 == we haven't registered a destructor, so do
54 // so now.
55 0 => {
56 $crate::thread::local_impl::Key::<$t>::register_dtor(
57 VAL.get() as *mut $crate::primitive::u8,
58 destroy,
59 );
60 STATE.set(1);
61 $crate::option::Option::Some(&*VAL.get())
62 }
63 // 1 == the destructor is registered and the value
64 // is valid, so return the pointer.
65 1 => $crate::option::Option::Some(&*VAL.get()),
66 // otherwise the destructor has already run, so we
67 // can't give access.
68 _ => $crate::option::Option::None,
69 }
70 }
71 }
72
73 unsafe {
74 $crate::thread::LocalKey::new(__getit)
75 }
76 }},
77
78 // used to generate the `LocalKey` value for `thread_local!`
79 (@key $t:ty, $init:expr) => {
80 {
81 #[inline]
82 fn __init() -> $t { $init }
83
84 #[inline]
85 unsafe fn __getit(
86 init: $crate::option::Option<&mut $crate::option::Option<$t>>,
87 ) -> $crate::option::Option<&'static $t> {
88 #[thread_local]
89 static __KEY: $crate::thread::local_impl::Key<$t> =
90 $crate::thread::local_impl::Key::<$t>::new();
91
92 unsafe {
93 __KEY.get(move || {
94 if let $crate::option::Option::Some(init) = init {
95 if let $crate::option::Option::Some(value) = init.take() {
96 return value;
97 }
98 if $crate::cfg!(debug_assertions) {
99 $crate::unreachable!("missing default value");
100 }
101 }
102 __init()
103 })
104 }
105 }
106
107 unsafe {
108 $crate::thread::LocalKey::new(__getit)
109 }
110 }
111 },
112 ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
113 $(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
114 $crate::thread::local_impl::thread_local_inner!(@key $t, $($init)*);
115 },
116}
117
118#[derive(Copy, Clone)]
119enum DtorState {
120 Unregistered,
121 Registered,
122 RunningOrHasRun,
123}
124
125// This data structure has been carefully constructed so that the fast path
126// only contains one branch on x86. That optimization is necessary to avoid
127// duplicated tls lookups on OSX.
128//
129// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
130pub struct Key<T> {
131 // If `LazyKeyInner::get` returns `None`, that indicates either:
132 // * The value has never been initialized
133 // * The value is being recursively initialized
134 // * The value has already been destroyed or is being destroyed
135 // To determine which kind of `None`, check `dtor_state`.
136 //
137 // This is very optimizer friendly for the fast path - initialized but
138 // not yet dropped.
139 inner: LazyKeyInner<T>,
140
141 // Metadata to keep track of the state of the destructor. Remember that
142 // this variable is thread-local, not global.
143 dtor_state: Cell<DtorState>,
144}
145
146impl<T> fmt::Debug for Key<T> {
147 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
148 f.debug_struct(name:"Key").finish_non_exhaustive()
149 }
150}
151impl<T> Key<T> {
152 pub const fn new() -> Key<T> {
153 Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
154 }
155
156 // note that this is just a publicly-callable function only for the
157 // const-initialized form of thread locals, basically a way to call the
158 // free `register_dtor` function defined elsewhere in std.
159 pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
160 unsafe {
161 register_dtor(a, dtor);
162 }
163 }
164
165 pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
166 // SAFETY: See the definitions of `LazyKeyInner::get` and
167 // `try_initialize` for more information.
168 //
169 // The caller must ensure no mutable references are ever active to
170 // the inner cell or the inner T when this is called.
171 // The `try_initialize` is dependant on the passed `init` function
172 // for this.
173 unsafe {
174 match self.inner.get() {
175 Some(val) => Some(val),
176 None => self.try_initialize(init),
177 }
178 }
179 }
180
181 // `try_initialize` is only called once per fast thread local variable,
182 // except in corner cases where thread_local dtors reference other
183 // thread_local's, or it is being recursively initialized.
184 //
185 // Macos: Inlining this function can cause two `tlv_get_addr` calls to
186 // be performed for every call to `Key::get`.
187 // LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
188 #[inline(never)]
189 unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
190 // SAFETY: See comment above (this function doc).
191 if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } {
192 // SAFETY: See comment above (this function doc).
193 Some(unsafe { self.inner.initialize(init) })
194 } else {
195 None
196 }
197 }
198
199 // `try_register_dtor` is only called once per fast thread local
200 // variable, except in corner cases where thread_local dtors reference
201 // other thread_local's, or it is being recursively initialized.
202 unsafe fn try_register_dtor(&self) -> bool {
203 match self.dtor_state.get() {
204 DtorState::Unregistered => {
205 // SAFETY: dtor registration happens before initialization.
206 // Passing `self` as a pointer while using `destroy_value<T>`
207 // is safe because the function will build a pointer to a
208 // Key<T>, which is the type of self and so find the correct
209 // size.
210 unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
211 self.dtor_state.set(DtorState::Registered);
212 true
213 }
214 DtorState::Registered => {
215 // recursively initialized
216 true
217 }
218 DtorState::RunningOrHasRun => false,
219 }
220 }
221}
222
223unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
224 let ptr: *mut Key = ptr as *mut Key<T>;
225
226 // SAFETY:
227 //
228 // The pointer `ptr` has been built just above and comes from
229 // `try_register_dtor` where it is originally a Key<T> coming from `self`,
230 // making it non-NUL and of the correct type.
231 //
232 // Right before we run the user destructor be sure to set the
233 // `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
234 // causes future calls to `get` to run `try_initialize_drop` again,
235 // which will now fail, and return `None`.
236 //
237 // Wrap the call in a catch to ensure unwinding is caught in the event
238 // a panic takes place in a destructor.
239 if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe {
240 let value: Option = (*ptr).inner.take();
241 (*ptr).dtor_state.set(val:DtorState::RunningOrHasRun);
242 drop(value);
243 })) {
244 rtabort!("thread local panicked on drop");
245 }
246}
247