nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-25 00:03:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rollup merge of #108927 - Ayush1325:pal-cleanup, r=workingjubilee

Browse Source 
Move __thread_local_inner to sys

Move `__thread_local_inner` macro in `crate:🧵:local` to `crate::sys`. Initially, I was thinking about removing this macro completely, but I could not find a way to create the generic statics without macros, so in the end, I just moved to code around.

This probably will need a rebase once https://github.com/rust-lang/rust/pull/108917 is merged

r? ``@workingjubilee``
This commit is contained in:
Matthias Krüger 2023-03-11 12:55:43 +01:00 committed by GitHub
commit 5e52ada714
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 724 additions and 610 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
library/std/src/sys/common/mod.rs
View File

@ -12,6 +12,7 @@
pub mod alloc;
pub mod small_c_string;
pub mod thread_local;
#[cfg(test)]
mod tests;

276
library/std/src/sys/common/thread_local/fast_local.rs Normal file
View File

@ -0,0 +1,276 @@
#[doc(hidden)]
#[macro_export]
#[allow_internal_unstable(
thread_local_internals,
cfg_target_thread_local,
thread_local,
libstd_thread_internals
)]
#[allow_internal_unsafe]
macro_rules! __thread_local_inner {
// used to generate the `LocalKey` value for const-initialized thread locals
(@key $t:ty, const $init:expr) => {{
#[cfg_attr(not(windows), inline)] // see comments below
#[deny(unsafe_op_in_unsafe_fn)]
unsafe fn __getit(
_init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
const INIT_EXPR: $t = $init;
// If the platform has support for `#[thread_local]`, use it.
#[thread_local]
static mut VAL: $t = INIT_EXPR;
// If a dtor isn't needed we can do something "very raw" and
// just get going.
if !$crate::mem::needs_drop::<$t>() {
unsafe {
return $crate::option::Option::Some(&VAL)
}
}
// 0 == dtor not registered
// 1 == dtor registered, dtor not run
// 2 == dtor registered and is running or has run
#[thread_local]
static mut STATE: $crate::primitive::u8 = 0;
unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) {
let ptr = ptr as *mut $t;
unsafe {
$crate::debug_assert_eq!(STATE, 1);
STATE = 2;
$crate::ptr::drop_in_place(ptr);
}
}
unsafe {
match STATE {
// 0 == we haven't registered a destructor, so do
// so now.
0 => {
$crate::thread::__LocalKeyInner::<$t>::register_dtor(
$crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8,
destroy,
);
STATE = 1;
$crate::option::Option::Some(&VAL)
}
// 1 == the destructor is registered and the value
// is valid, so return the pointer.
1 => $crate::option::Option::Some(&VAL),
// otherwise the destructor has already run, so we
// can't give access.
_ => $crate::option::Option::None,
}
}
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}};
// used to generate the `LocalKey` value for `thread_local!`
(@key $t:ty, $init:expr) => {
{
#[inline]
fn __init() -> $t { $init }
// When reading this function you might ask "why is this inlined
// everywhere other than Windows?", and that's a very reasonable
// question to ask. The short story is that it segfaults rustc if
// this function is inlined. The longer story is that Windows looks
// to not support `extern` references to thread locals across DLL
// boundaries. This appears to at least not be supported in the ABI
// that LLVM implements.
//
// Because of this we never inline on Windows, but we do inline on
// other platforms (where external references to thread locals
// across DLLs are supported). A better fix for this would be to
// inline this function on Windows, but only for "statically linked"
// components. For example if two separately compiled rlibs end up
// getting linked into a DLL then it's fine to inline this function
// across that boundary. It's only not fine to inline this function
// across a DLL boundary. Unfortunately rustc doesn't currently
// have this sort of logic available in an attribute, and it's not
// clear that rustc is even equipped to answer this (it's more of a
// Cargo question kinda). This means that, unfortunately, Windows
// gets the pessimistic path for now where it's never inlined.
//
// The issue of "should enable on Windows sometimes" is #84933
#[cfg_attr(not(windows), inline)]
unsafe fn __getit(
init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
#[thread_local]
static __KEY: $crate::thread::__LocalKeyInner<$t> =
$crate::thread::__LocalKeyInner::<$t>::new();
// FIXME: remove the #[allow(...)] marker when macros don't
// raise warning for missing/extraneous unsafe blocks anymore.
// See https://github.com/rust-lang/rust/issues/74838.
#[allow(unused_unsafe)]
unsafe {
__KEY.get(move || {
if let $crate::option::Option::Some(init) = init {
if let $crate::option::Option::Some(value) = init.take() {
return value;
} else if $crate::cfg!(debug_assertions) {
$crate::unreachable!("missing default value");
}
}
__init()
})
}
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}
};
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
$crate::__thread_local_inner!(@key $t, $($init)*);
}
}
#[doc(hidden)]
pub mod fast {
use super::super::lazy::LazyKeyInner;
use crate::cell::Cell;
use crate::sys::thread_local_dtor::register_dtor;
use crate::{fmt, mem, panic};
#[derive(Copy, Clone)]
enum DtorState {
Unregistered,
Registered,
RunningOrHasRun,
}
// This data structure has been carefully constructed so that the fast path
// only contains one branch on x86. That optimization is necessary to avoid
// duplicated tls lookups on OSX.
//
// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
pub struct Key<T> {
// If `LazyKeyInner::get` returns `None`, that indicates either:
// * The value has never been initialized
// * The value is being recursively initialized
// * The value has already been destroyed or is being destroyed
// To determine which kind of `None`, check `dtor_state`.
//
// This is very optimizer friendly for the fast path - initialized but
// not yet dropped.
inner: LazyKeyInner<T>,
// Metadata to keep track of the state of the destructor. Remember that
// this variable is thread-local, not global.
dtor_state: Cell<DtorState>,
}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
impl<T> Key<T> {
pub const fn new() -> Key<T> {
Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
}
// note that this is just a publicly-callable function only for the
// const-initialized form of thread locals, basically a way to call the
// free `register_dtor` function defined elsewhere in std.
pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
unsafe {
register_dtor(a, dtor);
}
}
pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// SAFETY: See the definitions of `LazyKeyInner::get` and
// `try_initialize` for more information.
//
// The caller must ensure no mutable references are ever active to
// the inner cell or the inner T when this is called.
// The `try_initialize` is dependant on the passed `init` function
// for this.
unsafe {
match self.inner.get() {
Some(val) => Some(val),
None => self.try_initialize(init),
}
}
}
// `try_initialize` is only called once per fast thread local variable,
// except in corner cases where thread_local dtors reference other
// thread_local's, or it is being recursively initialized.
//
// Macos: Inlining this function can cause two `tlv_get_addr` calls to
// be performed for every call to `Key::get`.
// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
#[inline(never)]
unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// SAFETY: See comment above (this function doc).
if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } {
// SAFETY: See comment above (this function doc).
Some(unsafe { self.inner.initialize(init) })
} else {
None
}
}
// `try_register_dtor` is only called once per fast thread local
// variable, except in corner cases where thread_local dtors reference
// other thread_local's, or it is being recursively initialized.
unsafe fn try_register_dtor(&self) -> bool {
match self.dtor_state.get() {
DtorState::Unregistered => {
// SAFETY: dtor registration happens before initialization.
// Passing `self` as a pointer while using `destroy_value<T>`
// is safe because the function will build a pointer to a
// Key<T>, which is the type of self and so find the correct
// size.
unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
self.dtor_state.set(DtorState::Registered);
true
}
DtorState::Registered => {
// recursively initialized
true
}
DtorState::RunningOrHasRun => false,
}
}
}
unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
let ptr = ptr as *mut Key<T>;
// SAFETY:
//
// The pointer `ptr` has been built just above and comes from
// `try_register_dtor` where it is originally a Key<T> coming from `self`,
// making it non-NUL and of the correct type.
//
// Right before we run the user destructor be sure to set the
// `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
// causes future calls to `get` to run `try_initialize_drop` again,
// which will now fail, and return `None`.
//
// Wrap the call in a catch to ensure unwinding is caught in the event
// a panic takes place in a destructor.
if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe {
let value = (*ptr).inner.take();
(*ptr).dtor_state.set(DtorState::RunningOrHasRun);
drop(value);
})) {
rtabort!("thread local panicked on drop");
}
}
}

109
library/std/src/sys/common/thread_local/mod.rs Normal file
View File

@ -0,0 +1,109 @@
//! The following module declarations are outside cfg_if because the internal
//! `__thread_local_internal` macro does not seem to be exported properly when using cfg_if
#![unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")]
#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics")))))]
mod fast_local;
#[cfg(all(
not(target_thread_local),
not(all(target_family = "wasm", not(target_feature = "atomics")))
))]
mod os_local;
#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
mod static_local;
#[cfg(not(test))]
cfg_if::cfg_if! {
if #[cfg(all(target_family = "wasm", not(target_feature = "atomics")))] {
#[doc(hidden)]
pub use static_local::statik::Key;
} else if #[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics")))))] {
#[doc(hidden)]
pub use fast_local::fast::Key;
} else if #[cfg(all(not(target_thread_local), not(all(target_family = "wasm", not(target_feature = "atomics")))))] {
#[doc(hidden)]
pub use os_local::os::Key;
}
}
#[doc(hidden)]
#[cfg(test)]
pub use realstd::thread::__LocalKeyInner as Key;
mod lazy {
use crate::cell::UnsafeCell;
use crate::hint;
use crate::mem;
pub struct LazyKeyInner<T> {
inner: UnsafeCell<Option<T>>,
}
impl<T> LazyKeyInner<T> {
pub const fn new() -> LazyKeyInner<T> {
LazyKeyInner { inner: UnsafeCell::new(None) }
}
pub unsafe fn get(&self) -> Option<&'static T> {
// SAFETY: The caller must ensure no reference is ever handed out to
// the inner cell nor mutable reference to the Option<T> inside said
// cell. This make it safe to hand a reference, though the lifetime
// of 'static is itself unsafe, making the get method unsafe.
unsafe { (*self.inner.get()).as_ref() }
}
/// The caller must ensure that no reference is active: this method
/// needs unique access.
pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T {
// Execute the initialization up front, *then* move it into our slot,
// just in case initialization fails.
let value = init();
let ptr = self.inner.get();
// SAFETY:
//
// note that this can in theory just be `*ptr = Some(value)`, but due to
// the compiler will currently codegen that pattern with something like:
//
// ptr::drop_in_place(ptr)
// ptr::write(ptr, Some(value))
//
// Due to this pattern it's possible for the destructor of the value in
// `ptr` (e.g., if this is being recursively initialized) to re-access
// TLS, in which case there will be a `&` and `&mut` pointer to the same
// value (an aliasing violation). To avoid setting the "I'm running a
// destructor" flag we just use `mem::replace` which should sequence the
// operations a little differently and make this safe to call.
//
// The precondition also ensures that we are the only one accessing
// `self` at the moment so replacing is fine.
unsafe {
let _ = mem::replace(&mut *ptr, Some(value));
}
// SAFETY: With the call to `mem::replace` it is guaranteed there is
// a `Some` behind `ptr`, not a `None` so `unreachable_unchecked`
// will never be reached.
unsafe {
// After storing `Some` we want to get a reference to the contents of
// what we just stored. While we could use `unwrap` here and it should
// always work it empirically doesn't seem to always get optimized away,
// which means that using something like `try_with` can pull in
// panicking code and cause a large size bloat.
match *ptr {
Some(ref x) => x,
None => hint::unreachable_unchecked(),
}
}
}
/// The other methods hand out references while taking &self.
/// As such, callers of this method must ensure no `&` and `&mut` are
/// available and used at the same time.
#[allow(unused)]
pub unsafe fn take(&mut self) -> Option<T> {
// SAFETY: See doc comment for this method.
unsafe { (*self.inner.get()).take() }
}
}
}

217
library/std/src/sys/common/thread_local/os_local.rs Normal file
View File

@ -0,0 +1,217 @@
#[doc(hidden)]
#[macro_export]
#[allow_internal_unstable(
thread_local_internals,
cfg_target_thread_local,
thread_local,
libstd_thread_internals
)]
#[allow_internal_unsafe]
macro_rules! __thread_local_inner {
// used to generate the `LocalKey` value for const-initialized thread locals
(@key $t:ty, const $init:expr) => {{
#[cfg_attr(not(windows), inline)] // see comments below
#[deny(unsafe_op_in_unsafe_fn)]
unsafe fn __getit(
_init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
const INIT_EXPR: $t = $init;
// On platforms without `#[thread_local]` we fall back to the
// same implementation as below for os thread locals.
#[inline]
const fn __init() -> $t { INIT_EXPR }
static __KEY: $crate::thread::__LocalKeyInner<$t> =
$crate::thread::__LocalKeyInner::new();
#[allow(unused_unsafe)]
unsafe {
__KEY.get(move || {
if let $crate::option::Option::Some(init) = _init {
if let $crate::option::Option::Some(value) = init.take() {
return value;
} else if $crate::cfg!(debug_assertions) {
$crate::unreachable!("missing initial value");
}
}
__init()
})
}
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}};
// used to generate the `LocalKey` value for `thread_local!`
(@key $t:ty, $init:expr) => {
{
#[inline]
fn __init() -> $t { $init }
// When reading this function you might ask "why is this inlined
// everywhere other than Windows?", and that's a very reasonable
// question to ask. The short story is that it segfaults rustc if
// this function is inlined. The longer story is that Windows looks
// to not support `extern` references to thread locals across DLL
// boundaries. This appears to at least not be supported in the ABI
// that LLVM implements.
//
// Because of this we never inline on Windows, but we do inline on
// other platforms (where external references to thread locals
// across DLLs are supported). A better fix for this would be to
// inline this function on Windows, but only for "statically linked"
// components. For example if two separately compiled rlibs end up
// getting linked into a DLL then it's fine to inline this function
// across that boundary. It's only not fine to inline this function
// across a DLL boundary. Unfortunately rustc doesn't currently
// have this sort of logic available in an attribute, and it's not
// clear that rustc is even equipped to answer this (it's more of a
// Cargo question kinda). This means that, unfortunately, Windows
// gets the pessimistic path for now where it's never inlined.
//
// The issue of "should enable on Windows sometimes" is #84933
#[cfg_attr(not(windows), inline)]
unsafe fn __getit(
init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
static __KEY: $crate::thread::__LocalKeyInner<$t> =
$crate::thread::__LocalKeyInner::new();
// FIXME: remove the #[allow(...)] marker when macros don't
// raise warning for missing/extraneous unsafe blocks anymore.
// See https://github.com/rust-lang/rust/issues/74838.
#[allow(unused_unsafe)]
unsafe {
__KEY.get(move || {
if let $crate::option::Option::Some(init) = init {
if let $crate::option::Option::Some(value) = init.take() {
return value;
} else if $crate::cfg!(debug_assertions) {
$crate::unreachable!("missing default value");
}
}
__init()
})
}
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}
};
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
$crate::__thread_local_inner!(@key $t, $($init)*);
}
}
#[doc(hidden)]
pub mod os {
use super::super::lazy::LazyKeyInner;
use crate::cell::Cell;
use crate::sys_common::thread_local_key::StaticKey as OsStaticKey;
use crate::{fmt, marker, panic, ptr};
/// Use a regular global static to store this key; the state provided will then be
/// thread-local.
pub struct Key<T> {
// OS-TLS key that we'll use to key off.
os: OsStaticKey,
marker: marker::PhantomData<Cell<T>>,
}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
unsafe impl<T> Sync for Key<T> {}
struct Value<T: 'static> {
inner: LazyKeyInner<T>,
key: &'static Key<T>,
}
impl<T: 'static> Key<T> {
#[rustc_const_unstable(feature = "thread_local_internals", issue = "none")]
pub const fn new() -> Key<T> {
Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData }
}
/// It is a requirement for the caller to ensure that no mutable
/// reference is active when this method is called.
pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: See the documentation for this method.
let ptr = unsafe { self.os.get() as *mut Value<T> };
if ptr.addr() > 1 {
// SAFETY: the check ensured the pointer is safe (its destructor
// is not running) + it is coming from a trusted source (self).
if let Some(ref value) = unsafe { (*ptr).inner.get() } {
return Some(value);
}
}
// SAFETY: At this point we are sure we have no value and so
// initializing (or trying to) is safe.
unsafe { self.try_initialize(init) }
}
// `try_initialize` is only called once per os thread local variable,
// except in corner cases where thread_local dtors reference other
// thread_local's, or it is being recursively initialized.
unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: No mutable references are ever handed out meaning getting
// the value is ok.
let ptr = unsafe { self.os.get() as *mut Value<T> };
if ptr.addr() == 1 {
// destructor is running
return None;
}
let ptr = if ptr.is_null() {
// If the lookup returned null, we haven't initialized our own
// local copy, so do that now.
let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self }));
// SAFETY: At this point we are sure there is no value inside
// ptr so setting it will not affect anyone else.
unsafe {
self.os.set(ptr as *mut u8);
}
ptr
} else {
// recursive initialization
ptr
};
// SAFETY: ptr has been ensured as non-NUL just above an so can be
// dereferenced safely.
unsafe { Some((*ptr).inner.initialize(init)) }
}
}
unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) {
// SAFETY:
//
// The OS TLS ensures that this key contains a null value when this
// destructor starts to run. We set it back to a sentinel value of 1 to
// ensure that any future calls to `get` for this thread will return
// `None`.
//
// Note that to prevent an infinite loop we reset it back to null right
// before we return from the destructor ourselves.
//
// Wrap the call in a catch to ensure unwinding is caught in the event
// a panic takes place in a destructor.
if let Err(_) = panic::catch_unwind(|| unsafe {
let ptr = Box::from_raw(ptr as *mut Value<T>);
let key = ptr.key;
key.os.set(ptr::invalid_mut(1));
drop(ptr);
key.os.set(ptr::null_mut());
}) {
rtabort!("thread local panicked on drop");
}
}
}

115
library/std/src/sys/common/thread_local/static_local.rs Normal file
View File

@ -0,0 +1,115 @@
#[doc(hidden)]
#[macro_export]
#[allow_internal_unstable(
thread_local_internals,
cfg_target_thread_local,
thread_local,
libstd_thread_internals
)]
#[allow_internal_unsafe]
macro_rules! __thread_local_inner {
// used to generate the `LocalKey` value for const-initialized thread locals
(@key $t:ty, const $init:expr) => {{
#[inline] // see comments below
#[deny(unsafe_op_in_unsafe_fn)]
unsafe fn __getit(
_init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
const INIT_EXPR: $t = $init;
// wasm without atomics maps directly to `static mut`, and dtors
// aren't implemented because thread dtors aren't really a thing
// on wasm right now
//
// FIXME(#84224) this should come after the `target_thread_local`
// block.
static mut VAL: $t = INIT_EXPR;
unsafe { $crate::option::Option::Some(&VAL) }
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}};
// used to generate the `LocalKey` value for `thread_local!`
(@key $t:ty, $init:expr) => {
{
#[inline]
fn __init() -> $t { $init }
#[inline]
unsafe fn __getit(
init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
static __KEY: $crate::thread::__LocalKeyInner<$t> =
$crate::thread::__LocalKeyInner::new();
// FIXME: remove the #[allow(...)] marker when macros don't
// raise warning for missing/extraneous unsafe blocks anymore.
// See https://github.com/rust-lang/rust/issues/74838.
#[allow(unused_unsafe)]
unsafe {
__KEY.get(move || {
if let $crate::option::Option::Some(init) = init {
if let $crate::option::Option::Some(value) = init.take() {
return value;
} else if $crate::cfg!(debug_assertions) {
$crate::unreachable!("missing default value");
}
}
__init()
})
}
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}
};
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
$crate::__thread_local_inner!(@key $t, $($init)*);
}
}
/// On some targets like wasm there's no threads, so no need to generate
/// thread locals and we can instead just use plain statics!
#[doc(hidden)]
pub mod statik {
use super::super::lazy::LazyKeyInner;
use crate::fmt;
pub struct Key<T> {
inner: LazyKeyInner<T>,
}
unsafe impl<T> Sync for Key<T> {}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
impl<T> Key<T> {
pub const fn new() -> Key<T> {
Key { inner: LazyKeyInner::new() }
}
pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: The caller must ensure no reference is ever handed out to
// the inner cell nor mutable reference to the Option<T> inside said
// cell. This make it safe to hand a reference, though the lifetime
// of 'static is itself unsafe, making the get method unsafe.
let value = unsafe {
match self.inner.get() {
Some(ref value) => value,
None => self.inner.initialize(init),
}
};
Some(value)
}
}
}

567
library/std/src/thread/local.rs
View File

@ -173,200 +173,6 @@ macro_rules! thread_local {
);
}
#[doc(hidden)]
#[unstable(feature = "thread_local_internals", reason = "should not be necessary", issue = "none")]
#[macro_export]
#[allow_internal_unstable(thread_local_internals, cfg_target_thread_local, thread_local)]
#[allow_internal_unsafe]
macro_rules! __thread_local_inner {
// used to generate the `LocalKey` value for const-initialized thread locals
(@key $t:ty, const $init:expr) => {{
#[cfg_attr(not(windows), inline)] // see comments below
#[deny(unsafe_op_in_unsafe_fn)]
unsafe fn __getit(
_init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
const INIT_EXPR: $t = $init;
// wasm without atomics maps directly to `static mut`, and dtors
// aren't implemented because thread dtors aren't really a thing
// on wasm right now
//
// FIXME(#84224) this should come after the `target_thread_local`
// block.
#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
{
static mut VAL: $t = INIT_EXPR;
unsafe { $crate::option::Option::Some(&VAL) }
}
// If the platform has support for `#[thread_local]`, use it.
#[cfg(all(
target_thread_local,
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
{
#[thread_local]
static mut VAL: $t = INIT_EXPR;
// If a dtor isn't needed we can do something "very raw" and
// just get going.
if !$crate::mem::needs_drop::<$t>() {
unsafe {
return $crate::option::Option::Some(&VAL)
}
}
// 0 == dtor not registered
// 1 == dtor registered, dtor not run
// 2 == dtor registered and is running or has run
#[thread_local]
static mut STATE: $crate::primitive::u8 = 0;
unsafe extern "C" fn destroy(ptr: *mut $crate::primitive::u8) {
let ptr = ptr as *mut $t;
unsafe {
$crate::debug_assert_eq!(STATE, 1);
STATE = 2;
$crate::ptr::drop_in_place(ptr);
}
}
unsafe {
match STATE {
// 0 == we haven't registered a destructor, so do
// so now.
0 => {
$crate::thread::__FastLocalKeyInner::<$t>::register_dtor(
$crate::ptr::addr_of_mut!(VAL) as *mut $crate::primitive::u8,
destroy,
);
STATE = 1;
$crate::option::Option::Some(&VAL)
}
// 1 == the destructor is registered and the value
// is valid, so return the pointer.
1 => $crate::option::Option::Some(&VAL),
// otherwise the destructor has already run, so we
// can't give access.
_ => $crate::option::Option::None,
}
}
}
// On platforms without `#[thread_local]` we fall back to the
// same implementation as below for os thread locals.
#[cfg(all(
not(target_thread_local),
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
{
#[inline]
const fn __init() -> $t { INIT_EXPR }
static __KEY: $crate::thread::__OsLocalKeyInner<$t> =
$crate::thread::__OsLocalKeyInner::new();
#[allow(unused_unsafe)]
unsafe {
__KEY.get(move || {
if let $crate::option::Option::Some(init) = _init {
if let $crate::option::Option::Some(value) = init.take() {
return value;
} else if $crate::cfg!(debug_assertions) {
$crate::unreachable!("missing initial value");
}
}
__init()
})
}
}
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}};
// used to generate the `LocalKey` value for `thread_local!`
(@key $t:ty, $init:expr) => {
{
#[inline]
fn __init() -> $t { $init }
// When reading this function you might ask "why is this inlined
// everywhere other than Windows?", and that's a very reasonable
// question to ask. The short story is that it segfaults rustc if
// this function is inlined. The longer story is that Windows looks
// to not support `extern` references to thread locals across DLL
// boundaries. This appears to at least not be supported in the ABI
// that LLVM implements.
//
// Because of this we never inline on Windows, but we do inline on
// other platforms (where external references to thread locals
// across DLLs are supported). A better fix for this would be to
// inline this function on Windows, but only for "statically linked"
// components. For example if two separately compiled rlibs end up
// getting linked into a DLL then it's fine to inline this function
// across that boundary. It's only not fine to inline this function
// across a DLL boundary. Unfortunately rustc doesn't currently
// have this sort of logic available in an attribute, and it's not
// clear that rustc is even equipped to answer this (it's more of a
// Cargo question kinda). This means that, unfortunately, Windows
// gets the pessimistic path for now where it's never inlined.
//
// The issue of "should enable on Windows sometimes" is #84933
#[cfg_attr(not(windows), inline)]
unsafe fn __getit(
init: $crate::option::Option<&mut $crate::option::Option<$t>>,
) -> $crate::option::Option<&'static $t> {
#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
static __KEY: $crate::thread::__StaticLocalKeyInner<$t> =
$crate::thread::__StaticLocalKeyInner::new();
#[thread_local]
#[cfg(all(
target_thread_local,
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
static __KEY: $crate::thread::__FastLocalKeyInner<$t> =
$crate::thread::__FastLocalKeyInner::new();
#[cfg(all(
not(target_thread_local),
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
static __KEY: $crate::thread::__OsLocalKeyInner<$t> =
$crate::thread::__OsLocalKeyInner::new();
// FIXME: remove the #[allow(...)] marker when macros don't
// raise warning for missing/extraneous unsafe blocks anymore.
// See https://github.com/rust-lang/rust/issues/74838.
#[allow(unused_unsafe)]
unsafe {
__KEY.get(move || {
if let $crate::option::Option::Some(init) = init {
if let $crate::option::Option::Some(value) = init.take() {
return value;
} else if $crate::cfg!(debug_assertions) {
$crate::unreachable!("missing default value");
}
}
__init()
})
}
}
unsafe {
$crate::thread::LocalKey::new(__getit)
}
}
};
($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)*) => {
$(#[$attr])* $vis const $name: $crate::thread::LocalKey<$t> =
$crate::__thread_local_inner!(@key $t, $($init)*);
}
}
/// An error returned by [`LocalKey::try_with`](struct.LocalKey.html#method.try_with).
#[stable(feature = "thread_local_try_with", since = "1.26.0")]
#[non_exhaustive]
@ -779,376 +585,3 @@ impl<T: 'static> LocalKey<RefCell<T>> {
self.with(|cell| cell.replace(value))
}
}
mod lazy {
use crate::cell::UnsafeCell;
use crate::hint;
use crate::mem;
pub struct LazyKeyInner<T> {
inner: UnsafeCell<Option<T>>,
}
impl<T> LazyKeyInner<T> {
pub const fn new() -> LazyKeyInner<T> {
LazyKeyInner { inner: UnsafeCell::new(None) }
}
pub unsafe fn get(&self) -> Option<&'static T> {
// SAFETY: The caller must ensure no reference is ever handed out to
// the inner cell nor mutable reference to the Option<T> inside said
// cell. This make it safe to hand a reference, though the lifetime
// of 'static is itself unsafe, making the get method unsafe.
unsafe { (*self.inner.get()).as_ref() }
}
/// The caller must ensure that no reference is active: this method
/// needs unique access.
pub unsafe fn initialize<F: FnOnce() -> T>(&self, init: F) -> &'static T {
// Execute the initialization up front, *then* move it into our slot,
// just in case initialization fails.
let value = init();
let ptr = self.inner.get();
// SAFETY:
//
// note that this can in theory just be `*ptr = Some(value)`, but due to
// the compiler will currently codegen that pattern with something like:
//
// ptr::drop_in_place(ptr)
// ptr::write(ptr, Some(value))
//
// Due to this pattern it's possible for the destructor of the value in
// `ptr` (e.g., if this is being recursively initialized) to re-access
// TLS, in which case there will be a `&` and `&mut` pointer to the same
// value (an aliasing violation). To avoid setting the "I'm running a
// destructor" flag we just use `mem::replace` which should sequence the
// operations a little differently and make this safe to call.
//
// The precondition also ensures that we are the only one accessing
// `self` at the moment so replacing is fine.
unsafe {
let _ = mem::replace(&mut *ptr, Some(value));
}
// SAFETY: With the call to `mem::replace` it is guaranteed there is
// a `Some` behind `ptr`, not a `None` so `unreachable_unchecked`
// will never be reached.
unsafe {
// After storing `Some` we want to get a reference to the contents of
// what we just stored. While we could use `unwrap` here and it should
// always work it empirically doesn't seem to always get optimized away,
// which means that using something like `try_with` can pull in
// panicking code and cause a large size bloat.
match *ptr {
Some(ref x) => x,
None => hint::unreachable_unchecked(),
}
}
}
/// The other methods hand out references while taking &self.
/// As such, callers of this method must ensure no `&` and `&mut` are
/// available and used at the same time.
#[allow(unused)]
pub unsafe fn take(&mut self) -> Option<T> {
// SAFETY: See doc comment for this method.
unsafe { (*self.inner.get()).take() }
}
}
}
/// On some targets like wasm there's no threads, so no need to generate
/// thread locals and we can instead just use plain statics!
#[doc(hidden)]
#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
pub mod statik {
use super::lazy::LazyKeyInner;
use crate::fmt;
pub struct Key<T> {
inner: LazyKeyInner<T>,
}
unsafe impl<T> Sync for Key<T> {}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
impl<T> Key<T> {
pub const fn new() -> Key<T> {
Key { inner: LazyKeyInner::new() }
}
pub unsafe fn get(&self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: The caller must ensure no reference is ever handed out to
// the inner cell nor mutable reference to the Option<T> inside said
// cell. This make it safe to hand a reference, though the lifetime
// of 'static is itself unsafe, making the get method unsafe.
let value = unsafe {
match self.inner.get() {
Some(ref value) => value,
None => self.inner.initialize(init),
}
};
Some(value)
}
}
}
#[doc(hidden)]
#[cfg(all(target_thread_local, not(all(target_family = "wasm", not(target_feature = "atomics"))),))]
pub mod fast {
use super::lazy::LazyKeyInner;
use crate::cell::Cell;
use crate::sys::thread_local_dtor::register_dtor;
use crate::{fmt, mem, panic};
#[derive(Copy, Clone)]
enum DtorState {
Unregistered,
Registered,
RunningOrHasRun,
}
// This data structure has been carefully constructed so that the fast path
// only contains one branch on x86. That optimization is necessary to avoid
// duplicated tls lookups on OSX.
//
// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
pub struct Key<T> {
// If `LazyKeyInner::get` returns `None`, that indicates either:
// * The value has never been initialized
// * The value is being recursively initialized
// * The value has already been destroyed or is being destroyed
// To determine which kind of `None`, check `dtor_state`.
//
// This is very optimizer friendly for the fast path - initialized but
// not yet dropped.
inner: LazyKeyInner<T>,
// Metadata to keep track of the state of the destructor. Remember that
// this variable is thread-local, not global.
dtor_state: Cell<DtorState>,
}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
impl<T> Key<T> {
pub const fn new() -> Key<T> {
Key { inner: LazyKeyInner::new(), dtor_state: Cell::new(DtorState::Unregistered) }
}
// note that this is just a publicly-callable function only for the
// const-initialized form of thread locals, basically a way to call the
// free `register_dtor` function defined elsewhere in std.
pub unsafe fn register_dtor(a: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) {
unsafe {
register_dtor(a, dtor);
}
}
pub unsafe fn get<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// SAFETY: See the definitions of `LazyKeyInner::get` and
// `try_initialize` for more information.
//
// The caller must ensure no mutable references are ever active to
// the inner cell or the inner T when this is called.
// The `try_initialize` is dependant on the passed `init` function
// for this.
unsafe {
match self.inner.get() {
Some(val) => Some(val),
None => self.try_initialize(init),
}
}
}
// `try_initialize` is only called once per fast thread local variable,
// except in corner cases where thread_local dtors reference other
// thread_local's, or it is being recursively initialized.
//
// Macos: Inlining this function can cause two `tlv_get_addr` calls to
// be performed for every call to `Key::get`.
// LLVM issue: https://bugs.llvm.org/show_bug.cgi?id=41722
#[inline(never)]
unsafe fn try_initialize<F: FnOnce() -> T>(&self, init: F) -> Option<&'static T> {
// SAFETY: See comment above (this function doc).
if !mem::needs_drop::<T>() || unsafe { self.try_register_dtor() } {
// SAFETY: See comment above (this function doc).
Some(unsafe { self.inner.initialize(init) })
} else {
None
}
}
// `try_register_dtor` is only called once per fast thread local
// variable, except in corner cases where thread_local dtors reference
// other thread_local's, or it is being recursively initialized.
unsafe fn try_register_dtor(&self) -> bool {
match self.dtor_state.get() {
DtorState::Unregistered => {
// SAFETY: dtor registration happens before initialization.
// Passing `self` as a pointer while using `destroy_value<T>`
// is safe because the function will build a pointer to a
// Key<T>, which is the type of self and so find the correct
// size.
unsafe { register_dtor(self as *const _ as *mut u8, destroy_value::<T>) };
self.dtor_state.set(DtorState::Registered);
true
}
DtorState::Registered => {
// recursively initialized
true
}
DtorState::RunningOrHasRun => false,
}
}
}
unsafe extern "C" fn destroy_value<T>(ptr: *mut u8) {
let ptr = ptr as *mut Key<T>;
// SAFETY:
//
// The pointer `ptr` has been built just above and comes from
// `try_register_dtor` where it is originally a Key<T> coming from `self`,
// making it non-NUL and of the correct type.
//
// Right before we run the user destructor be sure to set the
// `Option<T>` to `None`, and `dtor_state` to `RunningOrHasRun`. This
// causes future calls to `get` to run `try_initialize_drop` again,
// which will now fail, and return `None`.
//
// Wrap the call in a catch to ensure unwinding is caught in the event
// a panic takes place in a destructor.
if let Err(_) = panic::catch_unwind(panic::AssertUnwindSafe(|| unsafe {
let value = (*ptr).inner.take();
(*ptr).dtor_state.set(DtorState::RunningOrHasRun);
drop(value);
})) {
rtabort!("thread local panicked on drop");
}
}
}
#[doc(hidden)]
#[cfg(all(
not(target_thread_local),
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
pub mod os {
use super::lazy::LazyKeyInner;
use crate::cell::Cell;
use crate::sys_common::thread_local_key::StaticKey as OsStaticKey;
use crate::{fmt, marker, panic, ptr};
/// Use a regular global static to store this key; the state provided will then be
/// thread-local.
pub struct Key<T> {
// OS-TLS key that we'll use to key off.
os: OsStaticKey,
marker: marker::PhantomData<Cell<T>>,
}
impl<T> fmt::Debug for Key<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Key").finish_non_exhaustive()
}
}
unsafe impl<T> Sync for Key<T> {}
struct Value<T: 'static> {
inner: LazyKeyInner<T>,
key: &'static Key<T>,
}
impl<T: 'static> Key<T> {
#[rustc_const_unstable(feature = "thread_local_internals", issue = "none")]
pub const fn new() -> Key<T> {
Key { os: OsStaticKey::new(Some(destroy_value::<T>)), marker: marker::PhantomData }
}
/// It is a requirement for the caller to ensure that no mutable
/// reference is active when this method is called.
pub unsafe fn get(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: See the documentation for this method.
let ptr = unsafe { self.os.get() as *mut Value<T> };
if ptr.addr() > 1 {
// SAFETY: the check ensured the pointer is safe (its destructor
// is not running) + it is coming from a trusted source (self).
if let Some(ref value) = unsafe { (*ptr).inner.get() } {
return Some(value);
}
}
// SAFETY: At this point we are sure we have no value and so
// initializing (or trying to) is safe.
unsafe { self.try_initialize(init) }
}
// `try_initialize` is only called once per os thread local variable,
// except in corner cases where thread_local dtors reference other
// thread_local's, or it is being recursively initialized.
unsafe fn try_initialize(&'static self, init: impl FnOnce() -> T) -> Option<&'static T> {
// SAFETY: No mutable references are ever handed out meaning getting
// the value is ok.
let ptr = unsafe { self.os.get() as *mut Value<T> };
if ptr.addr() == 1 {
// destructor is running
return None;
}
let ptr = if ptr.is_null() {
// If the lookup returned null, we haven't initialized our own
// local copy, so do that now.
let ptr = Box::into_raw(Box::new(Value { inner: LazyKeyInner::new(), key: self }));
// SAFETY: At this point we are sure there is no value inside
// ptr so setting it will not affect anyone else.
unsafe {
self.os.set(ptr as *mut u8);
}
ptr
} else {
// recursive initialization
ptr
};
// SAFETY: ptr has been ensured as non-NUL just above an so can be
// dereferenced safely.
unsafe { Some((*ptr).inner.initialize(init)) }
}
}
unsafe extern "C" fn destroy_value<T: 'static>(ptr: *mut u8) {
// SAFETY:
//
// The OS TLS ensures that this key contains a null value when this
// destructor starts to run. We set it back to a sentinel value of 1 to
// ensure that any future calls to `get` for this thread will return
// `None`.
//
// Note that to prevent an infinite loop we reset it back to null right
// before we return from the destructor ourselves.
//
// Wrap the call in a catch to ensure unwinding is caught in the event
// a panic takes place in a destructor.
if let Err(_) = panic::catch_unwind(|| unsafe {
let ptr = Box::from_raw(ptr as *mut Value<T>);
let key = ptr.key;
key.os.set(ptr::invalid_mut(1));
drop(ptr);
key.os.set(ptr::null_mut());
}) {
rtabort!("thread local panicked on drop");
}
}
}

37
library/std/src/thread/mod.rs
View File

@ -203,44 +203,9 @@ pub use self::local::{AccessError, LocalKey};
// by the elf linker. "static" is for single-threaded platforms where a global
// static is sufficient.
#[unstable(feature = "libstd_thread_internals", issue = "none")]
#[cfg(not(test))]
#[cfg(all(
target_thread_local,
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
#[doc(hidden)]
pub use self::local::fast::Key as __FastLocalKeyInner;
// when building for tests, use real std's type
#[unstable(feature = "libstd_thread_internals", issue = "none")]
#[cfg(test)]
#[cfg(all(
target_thread_local,
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
pub use realstd::thread::__FastLocalKeyInner;
#[unstable(feature = "libstd_thread_internals", issue = "none")]
#[cfg(not(test))]
#[cfg(all(
not(target_thread_local),
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
#[doc(hidden)]
pub use self::local::os::Key as __OsLocalKeyInner;
// when building for tests, use real std's type
#[unstable(feature = "libstd_thread_internals", issue = "none")]
#[cfg(test)]
#[cfg(all(
not(target_thread_local),
not(all(target_family = "wasm", not(target_feature = "atomics"))),
))]
pub use realstd::thread::__OsLocalKeyInner;
#[unstable(feature = "libstd_thread_internals", issue = "none")]
#[cfg(all(target_family = "wasm", not(target_feature = "atomics")))]
#[doc(hidden)]
pub use self::local::statik::Key as __StaticLocalKeyInner;
pub use crate::sys::common::thread_local::Key as __LocalKeyInner;
////////////////////////////////////////////////////////////////////////////////
// Builder

2
src/tools/tidy/src/pal.rs
View File

@ -62,8 +62,6 @@ const EXCEPTION_PATHS: &[&str] = &[
"library/std/src/panic.rs", // fuchsia-specific panic backtrace handling
"library/std/src/personality.rs",
"library/std/src/personality/",
"library/std/src/thread/mod.rs",
"library/std/src/thread/local.rs",
];
pub fn check(path: &Path, bad: &mut bool) {

2
tests/ui/threads-sendsync/issue-43733-2.rs
View File

@ -21,7 +21,7 @@ impl<T> Key<T> {
}
#[cfg(target_thread_local)]
use std::thread::__FastLocalKeyInner as Key;
use std::thread::__LocalKeyInner as Key;
static __KEY: Key<()> = Key::new();
//~^ ERROR `UnsafeCell<Option<()>>` cannot be shared between threads

8
tests/ui/threads-sendsync/issue-43733.rs
View File

@ -1,8 +1,8 @@
// ignore-wasm32
// revisions: mir thir
// [thir]compile-flags: -Z thir-unsafeck
// normalize-stderr-test: "__FastLocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get"
// normalize-stderr-test: "__OsLocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get"
// normalize-stderr-test: "__LocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get"
// normalize-stderr-test: "__LocalKeyInner::<T>::get" -> "$$LOCALKEYINNER::<T>::get"
#![feature(thread_local)]
#![feature(cfg_target_thread_local, thread_local_internals)]
@ -12,10 +12,10 @@ type Foo = std::cell::RefCell<String>;
#[cfg(target_thread_local)]
#[thread_local]
static __KEY: std::thread::__FastLocalKeyInner<Foo> = std::thread::__FastLocalKeyInner::new();
static __KEY: std::thread::__LocalKeyInner<Foo> = std::thread::__LocalKeyInner::new();
#[cfg(not(target_thread_local))]
static __KEY: std::thread::__OsLocalKeyInner<Foo> = std::thread::__OsLocalKeyInner::new();
static __KEY: std::thread::__LocalKeyInner<Foo> = std::thread::__LocalKeyInner::new();
fn __getit(_: Option<&mut Option<RefCell<String>>>) -> std::option::Option<&'static Foo> {
__KEY.get(Default::default)