std\sys\thread_local\key/windows.rs
1//! Implementation of `LazyKey` for Windows.
2//!
3//! Windows has no native support for running destructors so we manage our own
4//! list of destructors to keep track of how to destroy keys. We then install a
5//! callback later to get invoked whenever a thread exits, running all
6//! appropriate destructors (see the [`guard`](guard) module documentation).
7//!
8//! This will likely need to be improved over time, but this module attempts a
9//! "poor man's" destructor callback system. Once we've got a list of what to
10//! run, we iterate over all keys, check their values, and then run destructors
11//! if the values turn out to be non null (setting them to null just beforehand).
12//! We do this a few times in a loop to basically match Unix semantics. If we
13//! don't reach a fixed point after a short while then we just inevitably leak
14//! something.
15//!
16//! The list is implemented as an atomic single-linked list of `LazyKey`s and
17//! does not support unregistration. Unfortunately, this means that we cannot
18//! use racy initialization for creating the keys in `LazyKey`, as that could
19//! result in destructors being missed. Hence, we synchronize the creation of
20//! keys with destructors through [`INIT_ONCE`](c::INIT_ONCE) (`std`'s
21//! [`Once`](crate::sync::Once) cannot be used since it might use TLS itself).
22//! For keys without destructors, racy initialization suffices.
23
24// FIXME: investigate using a fixed-size array instead, as the maximum number
25// of keys is [limited to 1088](https://learn.microsoft.com/en-us/windows/win32/ProcThread/thread-local-storage).
26
27use crate::cell::UnsafeCell;
28use crate::ptr;
29use crate::sync::atomic::Ordering::{AcqRel, Acquire, Relaxed, Release};
30use crate::sync::atomic::{Atomic, AtomicPtr, AtomicU32};
31use crate::sys::c;
32use crate::sys::thread_local::guard;
33
34pub type Key = u32;
35type Dtor = unsafe extern "C" fn(*mut u8);
36
37pub struct LazyKey {
38 /// The key value shifted up by one. Since TLS_OUT_OF_INDEXES == u32::MAX
39 /// is not a valid key value, this allows us to use zero as sentinel value
40 /// without risking overflow.
41 key: Atomic<Key>,
42 dtor: Option<Dtor>,
43 next: Atomic<*mut LazyKey>,
44 /// Currently, destructors cannot be unregistered, so we cannot use racy
45 /// initialization for keys. Instead, we need synchronize initialization.
46 /// Use the Windows-provided `Once` since it does not require TLS.
47 once: UnsafeCell<c::INIT_ONCE>,
48}
49
50impl LazyKey {
51 #[inline]
52 pub const fn new(dtor: Option<Dtor>) -> LazyKey {
53 LazyKey {
54 key: AtomicU32::new(0),
55 dtor,
56 next: AtomicPtr::new(ptr::null_mut()),
57 once: UnsafeCell::new(c::INIT_ONCE_STATIC_INIT),
58 }
59 }
60
61 #[inline]
62 pub fn force(&'static self) -> Key {
63 match self.key.load(Acquire) {
64 0 => unsafe { self.init() },
65 key => key - 1,
66 }
67 }
68
69 #[cold]
70 unsafe fn init(&'static self) -> Key {
71 if self.dtor.is_some() {
72 let mut pending = c::FALSE;
73 let r = unsafe {
74 c::InitOnceBeginInitialize(self.once.get(), 0, &mut pending, ptr::null_mut())
75 };
76 assert_eq!(r, c::TRUE);
77
78 if pending == c::FALSE {
79 // Some other thread initialized the key, load it.
80 self.key.load(Relaxed) - 1
81 } else {
82 let key = unsafe { c::TlsAlloc() };
83 if key == c::TLS_OUT_OF_INDEXES {
84 // Since we abort the process, there is no need to wake up
85 // the waiting threads. If this were a panic, the wakeup
86 // would need to occur first in order to avoid deadlock.
87 rtabort!("out of TLS indexes");
88 }
89
90 unsafe {
91 register_dtor(self);
92 }
93
94 // Release-storing the key needs to be the last thing we do.
95 // This is because in `fn key()`, other threads will do an acquire load of the key,
96 // and if that sees this write then it will entirely bypass the `InitOnce`. We thus
97 // need to establish synchronization through `key`. In particular that acquire load
98 // must happen-after the register_dtor above, to ensure the dtor actually runs!
99 self.key.store(key + 1, Release);
100
101 let r = unsafe { c::InitOnceComplete(self.once.get(), 0, ptr::null_mut()) };
102 debug_assert_eq!(r, c::TRUE);
103
104 key
105 }
106 } else {
107 // If there is no destructor to clean up, we can use racy initialization.
108
109 let key = unsafe { c::TlsAlloc() };
110 if key == c::TLS_OUT_OF_INDEXES {
111 rtabort!("out of TLS indexes");
112 }
113
114 match self.key.compare_exchange(0, key + 1, AcqRel, Acquire) {
115 Ok(_) => key,
116 Err(new) => unsafe {
117 // Some other thread completed initialization first, so destroy
118 // our key and use theirs.
119 let r = c::TlsFree(key);
120 debug_assert_eq!(r, c::TRUE);
121 new - 1
122 },
123 }
124 }
125 }
126}
127
128unsafe impl Send for LazyKey {}
129unsafe impl Sync for LazyKey {}
130
131#[inline]
132pub unsafe fn set(key: Key, val: *mut u8) {
133 let r = unsafe { c::TlsSetValue(key, val.cast()) };
134 debug_assert_eq!(r, c::TRUE);
135}
136
137#[inline]
138pub unsafe fn get(key: Key) -> *mut u8 {
139 unsafe { c::TlsGetValue(key).cast() }
140}
141
142static DTORS: Atomic<*mut LazyKey> = AtomicPtr::new(ptr::null_mut());
143
144/// Should only be called once per key, otherwise loops or breaks may occur in
145/// the linked list.
146unsafe fn register_dtor(key: &'static LazyKey) {
147 guard::enable();
148
149 let this = <*const LazyKey>::cast_mut(key);
150 // Use acquire ordering to pass along the changes done by the previously
151 // registered keys when we store the new head with release ordering.
152 let mut head = DTORS.load(Acquire);
153 loop {
154 key.next.store(head, Relaxed);
155 match DTORS.compare_exchange_weak(head, this, Release, Acquire) {
156 Ok(_) => break,
157 Err(new) => head = new,
158 }
159 }
160}
161
162/// This will and must only be run by the destructor callback in [`guard`].
163pub unsafe fn run_dtors() {
164 for _ in 0..5 {
165 let mut any_run = false;
166
167 // Use acquire ordering to observe key initialization.
168 let mut cur = DTORS.load(Acquire);
169 while !cur.is_null() {
170 let pre_key = unsafe { (*cur).key.load(Acquire) };
171 let dtor = unsafe { (*cur).dtor.unwrap() };
172 cur = unsafe { (*cur).next.load(Relaxed) };
173
174 // In LazyKey::init, we register the dtor before setting `key`.
175 // So if one thread's `run_dtors` races with another thread executing `init` on the same
176 // `LazyKey`, we can encounter a key of 0 here. That means this key was never
177 // initialized in this thread so we can safely skip it.
178 if pre_key == 0 {
179 continue;
180 }
181 // If this is non-zero, then via the `Acquire` load above we synchronized with
182 // everything relevant for this key. (It's not clear that this is needed, since the
183 // release-acquire pair on DTORS also establishes synchronization, but better safe than
184 // sorry.)
185 let key = pre_key - 1;
186
187 let ptr = unsafe { c::TlsGetValue(key) };
188 if !ptr.is_null() {
189 unsafe {
190 c::TlsSetValue(key, ptr::null_mut());
191 dtor(ptr as *mut _);
192 any_run = true;
193 }
194 }
195 }
196
197 if !any_run {
198 break;
199 }
200 }
201}