1 //! Implement thread-local storage. 2 3 use std::collections::btree_map::Entry as BTreeEntry; 4 use std::collections::hash_map::Entry as HashMapEntry; 5 use std::collections::BTreeMap; 6 7 use log::trace; 8 9 use rustc_data_structures::fx::FxHashMap; 10 use rustc_middle::ty; 11 use rustc_target::abi::{HasDataLayout, Size}; 12 use rustc_target::spec::abi::Abi; 13 14 use crate::*; 15 16 pub type TlsKey = u128; 17 18 #[derive(Clone, Debug)] 19 pub struct TlsEntry<'tcx> { 20 /// The data for this key. None is used to represent NULL. 21 /// (We normalize this early to avoid having to do a NULL-ptr-test each time we access the data.) 22 data: BTreeMap<ThreadId, Scalar<Tag>>, 23 dtor: Option<ty::Instance<'tcx>>, 24 } 25 26 #[derive(Clone, Debug)] 27 struct RunningDtorsState { 28 /// The last TlsKey used to retrieve a TLS destructor. `None` means that we 29 /// have not tried to retrieve a TLS destructor yet or that we already tried 30 /// all keys. 31 last_dtor_key: Option<TlsKey>, 32 } 33 34 #[derive(Debug)] 35 pub struct TlsData<'tcx> { 36 /// The Key to use for the next thread-local allocation. 37 next_key: TlsKey, 38 39 /// pthreads-style thread-local storage. 40 keys: BTreeMap<TlsKey, TlsEntry<'tcx>>, 41 42 /// A single per thread destructor of the thread local storage (that's how 43 /// things work on macOS) with a data argument. 44 macos_thread_dtors: BTreeMap<ThreadId, (ty::Instance<'tcx>, Scalar<Tag>)>, 45 46 /// State for currently running TLS dtors. If this map contains a key for a 47 /// specific thread, it means that we are in the "destruct" phase, during 48 /// which some operations are UB. 49 dtors_running: FxHashMap<ThreadId, RunningDtorsState>, 50 } 51 52 impl<'tcx> Default for TlsData<'tcx> { default() -> Self53 fn default() -> Self { 54 TlsData { 55 next_key: 1, // start with 1 as we must not use 0 on Windows 56 keys: Default::default(), 57 macos_thread_dtors: Default::default(), 58 dtors_running: Default::default(), 59 } 60 } 61 } 62 63 impl<'tcx> TlsData<'tcx> { 64 /// Generate a new TLS key with the given destructor. 65 /// `max_size` determines the integer size the key has to fit in. create_tls_key( &mut self, dtor: Option<ty::Instance<'tcx>>, max_size: Size, ) -> InterpResult<'tcx, TlsKey>66 pub fn create_tls_key( 67 &mut self, 68 dtor: Option<ty::Instance<'tcx>>, 69 max_size: Size, 70 ) -> InterpResult<'tcx, TlsKey> { 71 let new_key = self.next_key; 72 self.next_key += 1; 73 self.keys.try_insert(new_key, TlsEntry { data: Default::default(), dtor }).unwrap(); 74 trace!("New TLS key allocated: {} with dtor {:?}", new_key, dtor); 75 76 if max_size.bits() < 128 && new_key >= (1u128 << max_size.bits() as u128) { 77 throw_unsup_format!("we ran out of TLS key space"); 78 } 79 Ok(new_key) 80 } 81 delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx>82 pub fn delete_tls_key(&mut self, key: TlsKey) -> InterpResult<'tcx> { 83 match self.keys.remove(&key) { 84 Some(_) => { 85 trace!("TLS key {} removed", key); 86 Ok(()) 87 } 88 None => throw_ub_format!("removing a non-existig TLS key: {}", key), 89 } 90 } 91 load_tls( &self, key: TlsKey, thread_id: ThreadId, cx: &impl HasDataLayout, ) -> InterpResult<'tcx, Scalar<Tag>>92 pub fn load_tls( 93 &self, 94 key: TlsKey, 95 thread_id: ThreadId, 96 cx: &impl HasDataLayout, 97 ) -> InterpResult<'tcx, Scalar<Tag>> { 98 match self.keys.get(&key) { 99 Some(TlsEntry { data, .. }) => { 100 let value = data.get(&thread_id).copied(); 101 trace!("TLS key {} for thread {:?} loaded: {:?}", key, thread_id, value); 102 Ok(value.unwrap_or_else(|| Scalar::null_ptr(cx).into())) 103 } 104 None => throw_ub_format!("loading from a non-existing TLS key: {}", key), 105 } 106 } 107 store_tls( &mut self, key: TlsKey, thread_id: ThreadId, new_data: Scalar<Tag>, cx: &impl HasDataLayout, ) -> InterpResult<'tcx>108 pub fn store_tls( 109 &mut self, 110 key: TlsKey, 111 thread_id: ThreadId, 112 new_data: Scalar<Tag>, 113 cx: &impl HasDataLayout, 114 ) -> InterpResult<'tcx> { 115 match self.keys.get_mut(&key) { 116 Some(TlsEntry { data, .. }) => { 117 if new_data.to_machine_usize(cx)? != 0 { 118 trace!("TLS key {} for thread {:?} stored: {:?}", key, thread_id, new_data); 119 data.insert(thread_id, new_data); 120 } else { 121 trace!("TLS key {} for thread {:?} removed", key, thread_id); 122 data.remove(&thread_id); 123 } 124 Ok(()) 125 } 126 None => throw_ub_format!("storing to a non-existing TLS key: {}", key), 127 } 128 } 129 130 /// Set the thread wide destructor of the thread local storage for the given 131 /// thread. This function is used to implement `_tlv_atexit` shim on MacOS. 132 /// 133 /// Thread wide dtors are available only on MacOS. There is one destructor 134 /// per thread as can be guessed from the following comment in the 135 /// [`_tlv_atexit` 136 /// implementation](https://github.com/opensource-apple/dyld/blob/195030646877261f0c8c7ad8b001f52d6a26f514/src/threadLocalVariables.c#L389): 137 /// 138 /// // NOTE: this does not need locks because it only operates on current thread data set_macos_thread_dtor( &mut self, thread: ThreadId, dtor: ty::Instance<'tcx>, data: Scalar<Tag>, ) -> InterpResult<'tcx>139 pub fn set_macos_thread_dtor( 140 &mut self, 141 thread: ThreadId, 142 dtor: ty::Instance<'tcx>, 143 data: Scalar<Tag>, 144 ) -> InterpResult<'tcx> { 145 if self.dtors_running.contains_key(&thread) { 146 // UB, according to libstd docs. 147 throw_ub_format!( 148 "setting thread's local storage destructor while destructors are already running" 149 ); 150 } 151 if self.macos_thread_dtors.insert(thread, (dtor, data)).is_some() { 152 throw_unsup_format!( 153 "setting more than one thread local storage destructor for the same thread is not supported" 154 ); 155 } 156 Ok(()) 157 } 158 159 /// Returns a dtor, its argument and its index, if one is supposed to run. 160 /// `key` is the last dtors that was run; we return the *next* one after that. 161 /// 162 /// An optional destructor function may be associated with each key value. 163 /// At thread exit, if a key value has a non-NULL destructor pointer, 164 /// and the thread has a non-NULL value associated with that key, 165 /// the value of the key is set to NULL, and then the function pointed 166 /// to is called with the previously associated value as its sole argument. 167 /// The order of destructor calls is unspecified if more than one destructor 168 /// exists for a thread when it exits. 169 /// 170 /// If, after all the destructors have been called for all non-NULL values 171 /// with associated destructors, there are still some non-NULL values with 172 /// associated destructors, then the process is repeated. 173 /// If, after at least {PTHREAD_DESTRUCTOR_ITERATIONS} iterations of destructor 174 /// calls for outstanding non-NULL values, there are still some non-NULL values 175 /// with associated destructors, implementations may stop calling destructors, 176 /// or they may continue calling destructors until no non-NULL values with 177 /// associated destructors exist, even though this might result in an infinite loop. fetch_tls_dtor( &mut self, key: Option<TlsKey>, thread_id: ThreadId, ) -> Option<(ty::Instance<'tcx>, Scalar<Tag>, TlsKey)>178 fn fetch_tls_dtor( 179 &mut self, 180 key: Option<TlsKey>, 181 thread_id: ThreadId, 182 ) -> Option<(ty::Instance<'tcx>, Scalar<Tag>, TlsKey)> { 183 use std::ops::Bound::*; 184 185 let thread_local = &mut self.keys; 186 let start = match key { 187 Some(key) => Excluded(key), 188 None => Unbounded, 189 }; 190 for (&key, TlsEntry { data, dtor }) in thread_local.range_mut((start, Unbounded)) { 191 match data.entry(thread_id) { 192 BTreeEntry::Occupied(entry) => { 193 if let Some(dtor) = dtor { 194 // Set TLS data to NULL, and call dtor with old value. 195 let data_scalar = entry.remove(); 196 let ret = Some((*dtor, data_scalar, key)); 197 return ret; 198 } 199 } 200 BTreeEntry::Vacant(_) => {} 201 } 202 } 203 None 204 } 205 206 /// Set that dtors are running for `thread`. It is guaranteed not to change 207 /// the existing values stored in `dtors_running` for this thread. Returns 208 /// `true` if dtors for `thread` are already running. set_dtors_running_for_thread(&mut self, thread: ThreadId) -> bool209 fn set_dtors_running_for_thread(&mut self, thread: ThreadId) -> bool { 210 match self.dtors_running.entry(thread) { 211 HashMapEntry::Occupied(_) => true, 212 HashMapEntry::Vacant(entry) => { 213 // We cannot just do `self.dtors_running.insert` because that 214 // would overwrite `last_dtor_key` with `None`. 215 entry.insert(RunningDtorsState { last_dtor_key: None }); 216 false 217 } 218 } 219 } 220 221 /// Delete all TLS entries for the given thread. This function should be 222 /// called after all TLS destructors have already finished. delete_all_thread_tls(&mut self, thread_id: ThreadId)223 fn delete_all_thread_tls(&mut self, thread_id: ThreadId) { 224 for TlsEntry { data, .. } in self.keys.values_mut() { 225 data.remove(&thread_id); 226 } 227 } 228 } 229 230 impl<'mir, 'tcx: 'mir> EvalContextPrivExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {} 231 trait EvalContextPrivExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> { 232 /// Schedule TLS destructors for the main thread on Windows. The 233 /// implementation assumes that we do not support concurrency on Windows 234 /// yet. schedule_windows_tls_dtors(&mut self) -> InterpResult<'tcx>235 fn schedule_windows_tls_dtors(&mut self) -> InterpResult<'tcx> { 236 let this = self.eval_context_mut(); 237 let active_thread = this.get_active_thread(); 238 assert_eq!(this.get_total_thread_count(), 1, "concurrency on Windows is not supported"); 239 // Windows has a special magic linker section that is run on certain events. 240 // Instead of searching for that section and supporting arbitrary hooks in there 241 // (that would be basically https://github.com/rust-lang/miri/issues/450), 242 // we specifically look up the static in libstd that we know is placed 243 // in that section. 244 let thread_callback = this.eval_path_scalar(&[ 245 "std", 246 "sys", 247 "windows", 248 "thread_local_key", 249 "p_thread_callback", 250 ])?; 251 let thread_callback = 252 this.memory.get_fn(this.scalar_to_ptr(thread_callback))?.as_instance()?; 253 254 // The signature of this function is `unsafe extern "system" fn(h: c::LPVOID, dwReason: c::DWORD, pv: c::LPVOID)`. 255 let reason = this.eval_path_scalar(&["std", "sys", "windows", "c", "DLL_THREAD_DETACH"])?; 256 let ret_place = MPlaceTy::dangling(this.machine.layouts.unit).into(); 257 this.call_function( 258 thread_callback, 259 Abi::System { unwind: false }, 260 &[Scalar::null_ptr(this).into(), reason.into(), Scalar::null_ptr(this).into()], 261 Some(&ret_place), 262 StackPopCleanup::None { cleanup: true }, 263 )?; 264 265 this.enable_thread(active_thread); 266 Ok(()) 267 } 268 269 /// Schedule the MacOS thread destructor of the thread local storage to be 270 /// executed. Returns `true` if scheduled. 271 /// 272 /// Note: It is safe to call this function also on other Unixes. schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx, bool>273 fn schedule_macos_tls_dtor(&mut self) -> InterpResult<'tcx, bool> { 274 let this = self.eval_context_mut(); 275 let thread_id = this.get_active_thread(); 276 if let Some((instance, data)) = this.machine.tls.macos_thread_dtors.remove(&thread_id) { 277 trace!("Running macos dtor {:?} on {:?} at {:?}", instance, data, thread_id); 278 279 let ret_place = MPlaceTy::dangling(this.machine.layouts.unit).into(); 280 this.call_function( 281 instance, 282 Abi::C { unwind: false }, 283 &[data.into()], 284 Some(&ret_place), 285 StackPopCleanup::None { cleanup: true }, 286 )?; 287 288 // Enable the thread so that it steps through the destructor which 289 // we just scheduled. Since we deleted the destructor, it is 290 // guaranteed that we will schedule it again. The `dtors_running` 291 // flag will prevent the code from adding the destructor again. 292 this.enable_thread(thread_id); 293 Ok(true) 294 } else { 295 Ok(false) 296 } 297 } 298 299 /// Schedule a pthread TLS destructor. Returns `true` if found 300 /// a destructor to schedule, and `false` otherwise. schedule_next_pthread_tls_dtor(&mut self) -> InterpResult<'tcx, bool>301 fn schedule_next_pthread_tls_dtor(&mut self) -> InterpResult<'tcx, bool> { 302 let this = self.eval_context_mut(); 303 let active_thread = this.get_active_thread(); 304 305 assert!(this.has_terminated(active_thread), "running TLS dtors for non-terminated thread"); 306 // Fetch next dtor after `key`. 307 let last_key = this.machine.tls.dtors_running[&active_thread].last_dtor_key.clone(); 308 let dtor = match this.machine.tls.fetch_tls_dtor(last_key, active_thread) { 309 dtor @ Some(_) => dtor, 310 // We ran each dtor once, start over from the beginning. 311 None => this.machine.tls.fetch_tls_dtor(None, active_thread), 312 }; 313 if let Some((instance, ptr, key)) = dtor { 314 this.machine.tls.dtors_running.get_mut(&active_thread).unwrap().last_dtor_key = 315 Some(key); 316 trace!("Running TLS dtor {:?} on {:?} at {:?}", instance, ptr, active_thread); 317 assert!( 318 !ptr.to_machine_usize(this).unwrap() != 0, 319 "data can't be NULL when dtor is called!" 320 ); 321 322 let ret_place = MPlaceTy::dangling(this.machine.layouts.unit).into(); 323 this.call_function( 324 instance, 325 Abi::C { unwind: false }, 326 &[ptr.into()], 327 Some(&ret_place), 328 StackPopCleanup::None { cleanup: true }, 329 )?; 330 331 this.enable_thread(active_thread); 332 return Ok(true); 333 } 334 this.machine.tls.dtors_running.get_mut(&active_thread).unwrap().last_dtor_key = None; 335 336 Ok(false) 337 } 338 } 339 340 impl<'mir, 'tcx: 'mir> EvalContextExt<'mir, 'tcx> for crate::MiriEvalContext<'mir, 'tcx> {} 341 pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx> { 342 /// Schedule an active thread's TLS destructor to run on the active thread. 343 /// Note that this function does not run the destructors itself, it just 344 /// schedules them one by one each time it is called and reenables the 345 /// thread so that it can be executed normally by the main execution loop. 346 /// 347 /// Note: we consistently run TLS destructors for all threads, including the 348 /// main thread. However, it is not clear that we should run the TLS 349 /// destructors for the main thread. See issue: 350 /// https://github.com/rust-lang/rust/issues/28129. schedule_next_tls_dtor_for_active_thread(&mut self) -> InterpResult<'tcx>351 fn schedule_next_tls_dtor_for_active_thread(&mut self) -> InterpResult<'tcx> { 352 let this = self.eval_context_mut(); 353 let active_thread = this.get_active_thread(); 354 trace!("schedule_next_tls_dtor_for_active_thread on thread {:?}", active_thread); 355 356 if !this.machine.tls.set_dtors_running_for_thread(active_thread) { 357 // This is the first time we got asked to schedule a destructor. The 358 // Windows schedule destructor function must be called exactly once, 359 // this is why it is in this block. 360 if this.tcx.sess.target.os == "windows" { 361 // On Windows, we signal that the thread quit by starting the 362 // relevant function, reenabling the thread, and going back to 363 // the scheduler. 364 this.schedule_windows_tls_dtors()?; 365 return Ok(()); 366 } 367 } 368 // The remaining dtors make some progress each time around the scheduler loop, 369 // until they return `false` to indicate that they are done. 370 371 // The macOS thread wide destructor runs "before any TLS slots get 372 // freed", so do that first. 373 if this.schedule_macos_tls_dtor()? { 374 // We have scheduled a MacOS dtor to run on the thread. Execute it 375 // to completion and come back here. Scheduling a destructor 376 // destroys it, so we will not enter this branch again. 377 return Ok(()); 378 } 379 if this.schedule_next_pthread_tls_dtor()? { 380 // We have scheduled a pthread destructor and removed it from the 381 // destructors list. Run it to completion and come back here. 382 return Ok(()); 383 } 384 385 // All dtors done! 386 this.machine.tls.delete_all_thread_tls(active_thread); 387 this.thread_terminated()?; 388 389 Ok(()) 390 } 391 } 392