1 // SPDX-License-Identifier: Apache-2.0 OR MIT 2 3 //! This module provides the macros that actually implement the proc-macros `pin_data` and 4 //! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!` 5 //! macros. 6 //! 7 //! These macros should never be called directly, since they expect their input to be 8 //! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in 9 //! safe code! Use the public facing macros instead. 10 //! 11 //! This architecture has been chosen because the kernel does not yet have access to `syn` which 12 //! would make matters a lot easier for implementing these as proc-macros. 13 //! 14 //! # Macro expansion example 15 //! 16 //! This section is intended for readers trying to understand the macros in this module and the 17 //! `pin_init!` macros from `init.rs`. 18 //! 19 //! We will look at the following example: 20 //! 21 //! ```rust,ignore 22 //! # use kernel::init::*; 23 //! # use core::pin::Pin; 24 //! #[pin_data] 25 //! #[repr(C)] 26 //! struct Bar<T> { 27 //! #[pin] 28 //! t: T, 29 //! pub x: usize, 30 //! } 31 //! 32 //! impl<T> Bar<T> { 33 //! fn new(t: T) -> impl PinInit<Self> { 34 //! pin_init!(Self { t, x: 0 }) 35 //! } 36 //! } 37 //! 38 //! #[pin_data(PinnedDrop)] 39 //! struct Foo { 40 //! a: usize, 41 //! #[pin] 42 //! b: Bar<u32>, 43 //! } 44 //! 45 //! #[pinned_drop] 46 //! impl PinnedDrop for Foo { 47 //! fn drop(self: Pin<&mut Self>) { 48 //! pr_info!("{self:p} is getting dropped."); 49 //! } 50 //! } 51 //! 52 //! let a = 42; 53 //! let initializer = pin_init!(Foo { 54 //! a, 55 //! b <- Bar::new(36), 56 //! }); 57 //! ``` 58 //! 59 //! This example includes the most common and important features of the pin-init API. 60 //! 61 //! Below you can find individual section about the different macro invocations. Here are some 62 //! general things we need to take into account when designing macros: 63 //! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()` 64 //! this ensures that the correct item is used, since users could define their own `mod core {}` 65 //! and then their own `panic!` inside to execute arbitrary code inside of our macro. 66 //! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied 67 //! expressions inside of an `unsafe` block in the macro, because this would allow users to do 68 //! `unsafe` operations without an associated `unsafe` block. 69 //! 70 //! ## `#[pin_data]` on `Bar` 71 //! 72 //! This macro is used to specify which fields are structurally pinned and which fields are not. It 73 //! is placed on the struct definition and allows `#[pin]` to be placed on the fields. 74 //! 75 //! Here is the definition of `Bar` from our example: 76 //! 77 //! ```rust,ignore 78 //! # use kernel::init::*; 79 //! #[pin_data] 80 //! #[repr(C)] 81 //! struct Bar<T> { 82 //! #[pin] 83 //! t: T, 84 //! pub x: usize, 85 //! } 86 //! ``` 87 //! 88 //! This expands to the following code: 89 //! 90 //! ```rust,ignore 91 //! // Firstly the normal definition of the struct, attributes are preserved: 92 //! #[repr(C)] 93 //! struct Bar<T> { 94 //! t: T, 95 //! pub x: usize, 96 //! } 97 //! // Then an anonymous constant is defined, this is because we do not want any code to access the 98 //! // types that we define inside: 99 //! const _: () = { 100 //! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics, 101 //! // since we need to implement access functions for each field and thus need to know its 102 //! // type. 103 //! struct __ThePinData<T> { 104 //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 105 //! } 106 //! // We implement `Copy` for the pin-data struct, since all functions it defines will take 107 //! // `self` by value. 108 //! impl<T> ::core::clone::Clone for __ThePinData<T> { 109 //! fn clone(&self) -> Self { 110 //! *self 111 //! } 112 //! } 113 //! impl<T> ::core::marker::Copy for __ThePinData<T> {} 114 //! // For every field of `Bar`, the pin-data struct will define a function with the same name 115 //! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the 116 //! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field 117 //! // (if pinning is structural for the field, then `PinInit` otherwise `Init`). 118 //! #[allow(dead_code)] 119 //! impl<T> __ThePinData<T> { 120 //! unsafe fn t<E>( 121 //! self, 122 //! slot: *mut T, 123 //! // Since `t` is `#[pin]`, this is `PinInit`. 124 //! init: impl ::kernel::init::PinInit<T, E>, 125 //! ) -> ::core::result::Result<(), E> { 126 //! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } 127 //! } 128 //! pub unsafe fn x<E>( 129 //! self, 130 //! slot: *mut usize, 131 //! // Since `x` is not `#[pin]`, this is `Init`. 132 //! init: impl ::kernel::init::Init<usize, E>, 133 //! ) -> ::core::result::Result<(), E> { 134 //! unsafe { ::kernel::init::Init::__init(init, slot) } 135 //! } 136 //! } 137 //! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct 138 //! // that we constructed above. 139 //! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> { 140 //! type PinData = __ThePinData<T>; 141 //! unsafe fn __pin_data() -> Self::PinData { 142 //! __ThePinData { 143 //! __phantom: ::core::marker::PhantomData, 144 //! } 145 //! } 146 //! } 147 //! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data 148 //! // struct. This is important to ensure that no user can implement a rouge `__pin_data` 149 //! // function without using `unsafe`. 150 //! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> { 151 //! type Datee = Bar<T>; 152 //! } 153 //! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is 154 //! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned 155 //! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our 156 //! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist 157 //! // for two reasons: 158 //! // - `__phantom`: every generic must be used, since we cannot really know which generics 159 //! // are used, we declere all and then use everything here once. 160 //! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant 161 //! // over it. The lifetime is needed to work around the limitation that trait bounds must 162 //! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is 163 //! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler 164 //! // into accepting these bounds regardless. 165 //! #[allow(dead_code)] 166 //! struct __Unpin<'__pin, T> { 167 //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 168 //! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>, 169 //! // Our only `#[pin]` field is `t`. 170 //! t: T, 171 //! } 172 //! #[doc(hidden)] 173 //! impl<'__pin, T> ::core::marker::Unpin for Bar<T> 174 //! where 175 //! __Unpin<'__pin, T>: ::core::marker::Unpin, 176 //! {} 177 //! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users 178 //! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to 179 //! // UB with only safe code, so we disallow this by giving a trait implementation error using 180 //! // a direct impl and a blanket implementation. 181 //! trait MustNotImplDrop {} 182 //! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do 183 //! // (normally people want to know if a type has any kind of drop glue at all, here we want 184 //! // to know if it has any kind of custom drop glue, which is exactly what this bound does). 185 //! #[allow(drop_bounds)] 186 //! impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 187 //! impl<T> MustNotImplDrop for Bar<T> {} 188 //! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to 189 //! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed 190 //! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`. 191 //! #[allow(non_camel_case_types)] 192 //! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 193 //! impl< 194 //! T: ::kernel::init::PinnedDrop, 195 //! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 196 //! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {} 197 //! }; 198 //! ``` 199 //! 200 //! ## `pin_init!` in `impl Bar` 201 //! 202 //! This macro creates an pin-initializer for the given struct. It requires that the struct is 203 //! annotated by `#[pin_data]`. 204 //! 205 //! Here is the impl on `Bar` defining the new function: 206 //! 207 //! ```rust,ignore 208 //! impl<T> Bar<T> { 209 //! fn new(t: T) -> impl PinInit<Self> { 210 //! pin_init!(Self { t, x: 0 }) 211 //! } 212 //! } 213 //! ``` 214 //! 215 //! This expands to the following code: 216 //! 217 //! ```rust,ignore 218 //! impl<T> Bar<T> { 219 //! fn new(t: T) -> impl PinInit<Self> { 220 //! { 221 //! // We do not want to allow arbitrary returns, so we declare this type as the `Ok` 222 //! // return type and shadow it later when we insert the arbitrary user code. That way 223 //! // there will be no possibility of returning without `unsafe`. 224 //! struct __InitOk; 225 //! // Get the data about fields from the supplied type. 226 //! // - the function is unsafe, hence the unsafe block 227 //! // - we `use` the `HasPinData` trait in the block, it is only available in that 228 //! // scope. 229 //! let data = unsafe { 230 //! use ::kernel::init::__internal::HasPinData; 231 //! Self::__pin_data() 232 //! }; 233 //! // Ensure that `data` really is of type `PinData` and help with type inference: 234 //! let init = ::kernel::init::__internal::PinData::make_closure::< 235 //! _, 236 //! __InitOk, 237 //! ::core::convert::Infallible, 238 //! >(data, move |slot| { 239 //! { 240 //! // Shadow the structure so it cannot be used to return early. If a user 241 //! // tries to write `return Ok(__InitOk)`, then they get a type error, 242 //! // since that will refer to this struct instead of the one defined 243 //! // above. 244 //! struct __InitOk; 245 //! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`. 246 //! { 247 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) }; 248 //! } 249 //! // Since initialization could fail later (not in this case, since the 250 //! // error type is `Infallible`) we will need to drop this field if there 251 //! // is an error later. This `DropGuard` will drop the field when it gets 252 //! // dropped and has not yet been forgotten. 253 //! let __t_guard = unsafe { 254 //! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t)) 255 //! }; 256 //! // Expansion of `x: 0,`: 257 //! // Since this can be an arbitrary expression we cannot place it inside 258 //! // of the `unsafe` block, so we bind it here. 259 //! { 260 //! let x = 0; 261 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) }; 262 //! } 263 //! // We again create a `DropGuard`. 264 //! let __x_guard = unsafe { 265 //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x)) 266 //! }; 267 //! // Since initialization has successfully completed, we can now forget 268 //! // the guards. This is not `mem::forget`, since we only have 269 //! // `&DropGuard`. 270 //! ::core::mem::forget(__x_guard); 271 //! ::core::mem::forget(__t_guard); 272 //! // Here we use the type checker to ensure that every field has been 273 //! // initialized exactly once, since this is `if false` it will never get 274 //! // executed, but still type-checked. 275 //! // Additionally we abuse `slot` to automatically infer the correct type 276 //! // for the struct. This is also another check that every field is 277 //! // accessible from this scope. 278 //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 279 //! let _ = || { 280 //! unsafe { 281 //! ::core::ptr::write( 282 //! slot, 283 //! Self { 284 //! // We only care about typecheck finding every field 285 //! // here, the expression does not matter, just conjure 286 //! // one using `panic!()`: 287 //! t: ::core::panic!(), 288 //! x: ::core::panic!(), 289 //! }, 290 //! ); 291 //! }; 292 //! }; 293 //! } 294 //! // We leave the scope above and gain access to the previously shadowed 295 //! // `__InitOk` that we need to return. 296 //! Ok(__InitOk) 297 //! }); 298 //! // Change the return type from `__InitOk` to `()`. 299 //! let init = move | 300 //! slot, 301 //! | -> ::core::result::Result<(), ::core::convert::Infallible> { 302 //! init(slot).map(|__InitOk| ()) 303 //! }; 304 //! // Construct the initializer. 305 //! let init = unsafe { 306 //! ::kernel::init::pin_init_from_closure::< 307 //! _, 308 //! ::core::convert::Infallible, 309 //! >(init) 310 //! }; 311 //! init 312 //! } 313 //! } 314 //! } 315 //! ``` 316 //! 317 //! ## `#[pin_data]` on `Foo` 318 //! 319 //! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the 320 //! differences/new things in the expansion of the `Foo` definition: 321 //! 322 //! ```rust,ignore 323 //! #[pin_data(PinnedDrop)] 324 //! struct Foo { 325 //! a: usize, 326 //! #[pin] 327 //! b: Bar<u32>, 328 //! } 329 //! ``` 330 //! 331 //! This expands to the following code: 332 //! 333 //! ```rust,ignore 334 //! struct Foo { 335 //! a: usize, 336 //! b: Bar<u32>, 337 //! } 338 //! const _: () = { 339 //! struct __ThePinData { 340 //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 341 //! } 342 //! impl ::core::clone::Clone for __ThePinData { 343 //! fn clone(&self) -> Self { 344 //! *self 345 //! } 346 //! } 347 //! impl ::core::marker::Copy for __ThePinData {} 348 //! #[allow(dead_code)] 349 //! impl __ThePinData { 350 //! unsafe fn b<E>( 351 //! self, 352 //! slot: *mut Bar<u32>, 353 //! init: impl ::kernel::init::PinInit<Bar<u32>, E>, 354 //! ) -> ::core::result::Result<(), E> { 355 //! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) } 356 //! } 357 //! unsafe fn a<E>( 358 //! self, 359 //! slot: *mut usize, 360 //! init: impl ::kernel::init::Init<usize, E>, 361 //! ) -> ::core::result::Result<(), E> { 362 //! unsafe { ::kernel::init::Init::__init(init, slot) } 363 //! } 364 //! } 365 //! unsafe impl ::kernel::init::__internal::HasPinData for Foo { 366 //! type PinData = __ThePinData; 367 //! unsafe fn __pin_data() -> Self::PinData { 368 //! __ThePinData { 369 //! __phantom: ::core::marker::PhantomData, 370 //! } 371 //! } 372 //! } 373 //! unsafe impl ::kernel::init::__internal::PinData for __ThePinData { 374 //! type Datee = Foo; 375 //! } 376 //! #[allow(dead_code)] 377 //! struct __Unpin<'__pin> { 378 //! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 379 //! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>, 380 //! b: Bar<u32>, 381 //! } 382 //! #[doc(hidden)] 383 //! impl<'__pin> ::core::marker::Unpin for Foo 384 //! where 385 //! __Unpin<'__pin>: ::core::marker::Unpin, 386 //! {} 387 //! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to 388 //! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like 389 //! // before, instead we implement `Drop` here and delegate to `PinnedDrop`. 390 //! impl ::core::ops::Drop for Foo { 391 //! fn drop(&mut self) { 392 //! // Since we are getting dropped, no one else has a reference to `self` and thus we 393 //! // can assume that we never move. 394 //! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 395 //! // Create the unsafe token that proves that we are inside of a destructor, this 396 //! // type is only allowed to be created in a destructor. 397 //! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() }; 398 //! ::kernel::init::PinnedDrop::drop(pinned, token); 399 //! } 400 //! } 401 //! }; 402 //! ``` 403 //! 404 //! ## `#[pinned_drop]` on `impl PinnedDrop for Foo` 405 //! 406 //! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an 407 //! extra parameter that should not be used at all. The macro hides that parameter. 408 //! 409 //! Here is the `PinnedDrop` impl for `Foo`: 410 //! 411 //! ```rust,ignore 412 //! #[pinned_drop] 413 //! impl PinnedDrop for Foo { 414 //! fn drop(self: Pin<&mut Self>) { 415 //! pr_info!("{self:p} is getting dropped."); 416 //! } 417 //! } 418 //! ``` 419 //! 420 //! This expands to the following code: 421 //! 422 //! ```rust,ignore 423 //! // `unsafe`, full path and the token parameter are added, everything else stays the same. 424 //! unsafe impl ::kernel::init::PinnedDrop for Foo { 425 //! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) { 426 //! pr_info!("{self:p} is getting dropped."); 427 //! } 428 //! } 429 //! ``` 430 //! 431 //! ## `pin_init!` on `Foo` 432 //! 433 //! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion 434 //! of `pin_init!` on `Foo`: 435 //! 436 //! ```rust,ignore 437 //! let a = 42; 438 //! let initializer = pin_init!(Foo { 439 //! a, 440 //! b <- Bar::new(36), 441 //! }); 442 //! ``` 443 //! 444 //! This expands to the following code: 445 //! 446 //! ```rust,ignore 447 //! let a = 42; 448 //! let initializer = { 449 //! struct __InitOk; 450 //! let data = unsafe { 451 //! use ::kernel::init::__internal::HasPinData; 452 //! Foo::__pin_data() 453 //! }; 454 //! let init = ::kernel::init::__internal::PinData::make_closure::< 455 //! _, 456 //! __InitOk, 457 //! ::core::convert::Infallible, 458 //! >(data, move |slot| { 459 //! { 460 //! struct __InitOk; 461 //! { 462 //! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) }; 463 //! } 464 //! let __a_guard = unsafe { 465 //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a)) 466 //! }; 467 //! let init = Bar::new(36); 468 //! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? }; 469 //! let __b_guard = unsafe { 470 //! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b)) 471 //! }; 472 //! ::core::mem::forget(__b_guard); 473 //! ::core::mem::forget(__a_guard); 474 //! #[allow(unreachable_code, clippy::diverging_sub_expression)] 475 //! let _ = || { 476 //! unsafe { 477 //! ::core::ptr::write( 478 //! slot, 479 //! Foo { 480 //! a: ::core::panic!(), 481 //! b: ::core::panic!(), 482 //! }, 483 //! ); 484 //! }; 485 //! }; 486 //! } 487 //! Ok(__InitOk) 488 //! }); 489 //! let init = move | 490 //! slot, 491 //! | -> ::core::result::Result<(), ::core::convert::Infallible> { 492 //! init(slot).map(|__InitOk| ()) 493 //! }; 494 //! let init = unsafe { 495 //! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init) 496 //! }; 497 //! init 498 //! }; 499 //! ``` 500 501 /// Creates a `unsafe impl<...> PinnedDrop for $type` block. 502 /// 503 /// See [`PinnedDrop`] for more information. 504 #[doc(hidden)] 505 #[macro_export] 506 macro_rules! __pinned_drop { 507 ( 508 @impl_sig($($impl_sig:tt)*), 509 @impl_body( 510 $(#[$($attr:tt)*])* 511 fn drop($($sig:tt)*) { 512 $($inner:tt)* 513 } 514 ), 515 ) => { 516 unsafe $($impl_sig)* { 517 // Inherit all attributes and the type/ident tokens for the signature. 518 $(#[$($attr)*])* 519 fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) { 520 $($inner)* 521 } 522 } 523 } 524 } 525 526 /// This macro first parses the struct definition such that it separates pinned and not pinned 527 /// fields. Afterwards it declares the struct and implement the `PinData` trait safely. 528 #[doc(hidden)] 529 #[macro_export] 530 macro_rules! __pin_data { 531 // Proc-macro entry point, this is supplied by the proc-macro pre-parsing. 532 (parse_input: 533 @args($($pinned_drop:ident)?), 534 @sig( 535 $(#[$($struct_attr:tt)*])* 536 $vis:vis struct $name:ident 537 $(where $($whr:tt)*)? 538 ), 539 @impl_generics($($impl_generics:tt)*), 540 @ty_generics($($ty_generics:tt)*), 541 @decl_generics($($decl_generics:tt)*), 542 @body({ $($fields:tt)* }), 543 ) => { 544 // We now use token munching to iterate through all of the fields. While doing this we 545 // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user 546 // wants these to be structurally pinned. The rest of the fields are the 547 // 'not pinned fields'. Additionally we collect all fields, since we need them in the right 548 // order to declare the struct. 549 // 550 // In this call we also put some explaining comments for the parameters. 551 $crate::__pin_data!(find_pinned_fields: 552 // Attributes on the struct itself, these will just be propagated to be put onto the 553 // struct definition. 554 @struct_attrs($(#[$($struct_attr)*])*), 555 // The visibility of the struct. 556 @vis($vis), 557 // The name of the struct. 558 @name($name), 559 // The 'impl generics', the generics that will need to be specified on the struct inside 560 // of an `impl<$ty_generics>` block. 561 @impl_generics($($impl_generics)*), 562 // The 'ty generics', the generics that will need to be specified on the impl blocks. 563 @ty_generics($($ty_generics)*), 564 // The 'decl generics', the generics that need to be specified on the struct 565 // definition. 566 @decl_generics($($decl_generics)*), 567 // The where clause of any impl block and the declaration. 568 @where($($($whr)*)?), 569 // The remaining fields tokens that need to be processed. 570 // We add a `,` at the end to ensure correct parsing. 571 @fields_munch($($fields)* ,), 572 // The pinned fields. 573 @pinned(), 574 // The not pinned fields. 575 @not_pinned(), 576 // All fields. 577 @fields(), 578 // The accumulator containing all attributes already parsed. 579 @accum(), 580 // Contains `yes` or `` to indicate if `#[pin]` was found on the current field. 581 @is_pinned(), 582 // The proc-macro argument, this should be `PinnedDrop` or ``. 583 @pinned_drop($($pinned_drop)?), 584 ); 585 }; 586 (find_pinned_fields: 587 @struct_attrs($($struct_attrs:tt)*), 588 @vis($vis:vis), 589 @name($name:ident), 590 @impl_generics($($impl_generics:tt)*), 591 @ty_generics($($ty_generics:tt)*), 592 @decl_generics($($decl_generics:tt)*), 593 @where($($whr:tt)*), 594 // We found a PhantomPinned field, this should generally be pinned! 595 @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*), 596 @pinned($($pinned:tt)*), 597 @not_pinned($($not_pinned:tt)*), 598 @fields($($fields:tt)*), 599 @accum($($accum:tt)*), 600 // This field is not pinned. 601 @is_pinned(), 602 @pinned_drop($($pinned_drop:ident)?), 603 ) => { 604 ::core::compile_error!(concat!( 605 "The field `", 606 stringify!($field), 607 "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.", 608 )); 609 $crate::__pin_data!(find_pinned_fields: 610 @struct_attrs($($struct_attrs)*), 611 @vis($vis), 612 @name($name), 613 @impl_generics($($impl_generics)*), 614 @ty_generics($($ty_generics)*), 615 @decl_generics($($decl_generics)*), 616 @where($($whr)*), 617 @fields_munch($($rest)*), 618 @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,), 619 @not_pinned($($not_pinned)*), 620 @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,), 621 @accum(), 622 @is_pinned(), 623 @pinned_drop($($pinned_drop)?), 624 ); 625 }; 626 (find_pinned_fields: 627 @struct_attrs($($struct_attrs:tt)*), 628 @vis($vis:vis), 629 @name($name:ident), 630 @impl_generics($($impl_generics:tt)*), 631 @ty_generics($($ty_generics:tt)*), 632 @decl_generics($($decl_generics:tt)*), 633 @where($($whr:tt)*), 634 // We reached the field declaration. 635 @fields_munch($field:ident : $type:ty, $($rest:tt)*), 636 @pinned($($pinned:tt)*), 637 @not_pinned($($not_pinned:tt)*), 638 @fields($($fields:tt)*), 639 @accum($($accum:tt)*), 640 // This field is pinned. 641 @is_pinned(yes), 642 @pinned_drop($($pinned_drop:ident)?), 643 ) => { 644 $crate::__pin_data!(find_pinned_fields: 645 @struct_attrs($($struct_attrs)*), 646 @vis($vis), 647 @name($name), 648 @impl_generics($($impl_generics)*), 649 @ty_generics($($ty_generics)*), 650 @decl_generics($($decl_generics)*), 651 @where($($whr)*), 652 @fields_munch($($rest)*), 653 @pinned($($pinned)* $($accum)* $field: $type,), 654 @not_pinned($($not_pinned)*), 655 @fields($($fields)* $($accum)* $field: $type,), 656 @accum(), 657 @is_pinned(), 658 @pinned_drop($($pinned_drop)?), 659 ); 660 }; 661 (find_pinned_fields: 662 @struct_attrs($($struct_attrs:tt)*), 663 @vis($vis:vis), 664 @name($name:ident), 665 @impl_generics($($impl_generics:tt)*), 666 @ty_generics($($ty_generics:tt)*), 667 @decl_generics($($decl_generics:tt)*), 668 @where($($whr:tt)*), 669 // We reached the field declaration. 670 @fields_munch($field:ident : $type:ty, $($rest:tt)*), 671 @pinned($($pinned:tt)*), 672 @not_pinned($($not_pinned:tt)*), 673 @fields($($fields:tt)*), 674 @accum($($accum:tt)*), 675 // This field is not pinned. 676 @is_pinned(), 677 @pinned_drop($($pinned_drop:ident)?), 678 ) => { 679 $crate::__pin_data!(find_pinned_fields: 680 @struct_attrs($($struct_attrs)*), 681 @vis($vis), 682 @name($name), 683 @impl_generics($($impl_generics)*), 684 @ty_generics($($ty_generics)*), 685 @decl_generics($($decl_generics)*), 686 @where($($whr)*), 687 @fields_munch($($rest)*), 688 @pinned($($pinned)*), 689 @not_pinned($($not_pinned)* $($accum)* $field: $type,), 690 @fields($($fields)* $($accum)* $field: $type,), 691 @accum(), 692 @is_pinned(), 693 @pinned_drop($($pinned_drop)?), 694 ); 695 }; 696 (find_pinned_fields: 697 @struct_attrs($($struct_attrs:tt)*), 698 @vis($vis:vis), 699 @name($name:ident), 700 @impl_generics($($impl_generics:tt)*), 701 @ty_generics($($ty_generics:tt)*), 702 @decl_generics($($decl_generics:tt)*), 703 @where($($whr:tt)*), 704 // We found the `#[pin]` attr. 705 @fields_munch(#[pin] $($rest:tt)*), 706 @pinned($($pinned:tt)*), 707 @not_pinned($($not_pinned:tt)*), 708 @fields($($fields:tt)*), 709 @accum($($accum:tt)*), 710 @is_pinned($($is_pinned:ident)?), 711 @pinned_drop($($pinned_drop:ident)?), 712 ) => { 713 $crate::__pin_data!(find_pinned_fields: 714 @struct_attrs($($struct_attrs)*), 715 @vis($vis), 716 @name($name), 717 @impl_generics($($impl_generics)*), 718 @ty_generics($($ty_generics)*), 719 @decl_generics($($decl_generics)*), 720 @where($($whr)*), 721 @fields_munch($($rest)*), 722 // We do not include `#[pin]` in the list of attributes, since it is not actually an 723 // attribute that is defined somewhere. 724 @pinned($($pinned)*), 725 @not_pinned($($not_pinned)*), 726 @fields($($fields)*), 727 @accum($($accum)*), 728 // Set this to `yes`. 729 @is_pinned(yes), 730 @pinned_drop($($pinned_drop)?), 731 ); 732 }; 733 (find_pinned_fields: 734 @struct_attrs($($struct_attrs:tt)*), 735 @vis($vis:vis), 736 @name($name:ident), 737 @impl_generics($($impl_generics:tt)*), 738 @ty_generics($($ty_generics:tt)*), 739 @decl_generics($($decl_generics:tt)*), 740 @where($($whr:tt)*), 741 // We reached the field declaration with visibility, for simplicity we only munch the 742 // visibility and put it into `$accum`. 743 @fields_munch($fvis:vis $field:ident $($rest:tt)*), 744 @pinned($($pinned:tt)*), 745 @not_pinned($($not_pinned:tt)*), 746 @fields($($fields:tt)*), 747 @accum($($accum:tt)*), 748 @is_pinned($($is_pinned:ident)?), 749 @pinned_drop($($pinned_drop:ident)?), 750 ) => { 751 $crate::__pin_data!(find_pinned_fields: 752 @struct_attrs($($struct_attrs)*), 753 @vis($vis), 754 @name($name), 755 @impl_generics($($impl_generics)*), 756 @ty_generics($($ty_generics)*), 757 @decl_generics($($decl_generics)*), 758 @where($($whr)*), 759 @fields_munch($field $($rest)*), 760 @pinned($($pinned)*), 761 @not_pinned($($not_pinned)*), 762 @fields($($fields)*), 763 @accum($($accum)* $fvis), 764 @is_pinned($($is_pinned)?), 765 @pinned_drop($($pinned_drop)?), 766 ); 767 }; 768 (find_pinned_fields: 769 @struct_attrs($($struct_attrs:tt)*), 770 @vis($vis:vis), 771 @name($name:ident), 772 @impl_generics($($impl_generics:tt)*), 773 @ty_generics($($ty_generics:tt)*), 774 @decl_generics($($decl_generics:tt)*), 775 @where($($whr:tt)*), 776 // Some other attribute, just put it into `$accum`. 777 @fields_munch(#[$($attr:tt)*] $($rest:tt)*), 778 @pinned($($pinned:tt)*), 779 @not_pinned($($not_pinned:tt)*), 780 @fields($($fields:tt)*), 781 @accum($($accum:tt)*), 782 @is_pinned($($is_pinned:ident)?), 783 @pinned_drop($($pinned_drop:ident)?), 784 ) => { 785 $crate::__pin_data!(find_pinned_fields: 786 @struct_attrs($($struct_attrs)*), 787 @vis($vis), 788 @name($name), 789 @impl_generics($($impl_generics)*), 790 @ty_generics($($ty_generics)*), 791 @decl_generics($($decl_generics)*), 792 @where($($whr)*), 793 @fields_munch($($rest)*), 794 @pinned($($pinned)*), 795 @not_pinned($($not_pinned)*), 796 @fields($($fields)*), 797 @accum($($accum)* #[$($attr)*]), 798 @is_pinned($($is_pinned)?), 799 @pinned_drop($($pinned_drop)?), 800 ); 801 }; 802 (find_pinned_fields: 803 @struct_attrs($($struct_attrs:tt)*), 804 @vis($vis:vis), 805 @name($name:ident), 806 @impl_generics($($impl_generics:tt)*), 807 @ty_generics($($ty_generics:tt)*), 808 @decl_generics($($decl_generics:tt)*), 809 @where($($whr:tt)*), 810 // We reached the end of the fields, plus an optional additional comma, since we added one 811 // before and the user is also allowed to put a trailing comma. 812 @fields_munch($(,)?), 813 @pinned($($pinned:tt)*), 814 @not_pinned($($not_pinned:tt)*), 815 @fields($($fields:tt)*), 816 @accum(), 817 @is_pinned(), 818 @pinned_drop($($pinned_drop:ident)?), 819 ) => { 820 // Declare the struct with all fields in the correct order. 821 $($struct_attrs)* 822 $vis struct $name <$($decl_generics)*> 823 where $($whr)* 824 { 825 $($fields)* 826 } 827 828 // We put the rest into this const item, because it then will not be accessible to anything 829 // outside. 830 const _: () = { 831 // We declare this struct which will host all of the projection function for our type. 832 // it will be invariant over all generic parameters which are inherited from the 833 // struct. 834 $vis struct __ThePinData<$($impl_generics)*> 835 where $($whr)* 836 { 837 __phantom: ::core::marker::PhantomData< 838 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 839 >, 840 } 841 842 impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*> 843 where $($whr)* 844 { 845 fn clone(&self) -> Self { *self } 846 } 847 848 impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*> 849 where $($whr)* 850 {} 851 852 // Make all projection functions. 853 $crate::__pin_data!(make_pin_data: 854 @pin_data(__ThePinData), 855 @impl_generics($($impl_generics)*), 856 @ty_generics($($ty_generics)*), 857 @where($($whr)*), 858 @pinned($($pinned)*), 859 @not_pinned($($not_pinned)*), 860 ); 861 862 // SAFETY: We have added the correct projection functions above to `__ThePinData` and 863 // we also use the least restrictive generics possible. 864 unsafe impl<$($impl_generics)*> 865 $crate::init::__internal::HasPinData for $name<$($ty_generics)*> 866 where $($whr)* 867 { 868 type PinData = __ThePinData<$($ty_generics)*>; 869 870 unsafe fn __pin_data() -> Self::PinData { 871 __ThePinData { __phantom: ::core::marker::PhantomData } 872 } 873 } 874 875 unsafe impl<$($impl_generics)*> 876 $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*> 877 where $($whr)* 878 { 879 type Datee = $name<$($ty_generics)*>; 880 } 881 882 // This struct will be used for the unpin analysis. Since only structurally pinned 883 // fields are relevant whether the struct should implement `Unpin`. 884 #[allow(dead_code)] 885 struct __Unpin <'__pin, $($impl_generics)*> 886 where $($whr)* 887 { 888 __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>, 889 __phantom: ::core::marker::PhantomData< 890 fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*> 891 >, 892 // Only the pinned fields. 893 $($pinned)* 894 } 895 896 #[doc(hidden)] 897 impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*> 898 where 899 __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin, 900 $($whr)* 901 {} 902 903 // We need to disallow normal `Drop` implementation, the exact behavior depends on 904 // whether `PinnedDrop` was specified as the parameter. 905 $crate::__pin_data!(drop_prevention: 906 @name($name), 907 @impl_generics($($impl_generics)*), 908 @ty_generics($($ty_generics)*), 909 @where($($whr)*), 910 @pinned_drop($($pinned_drop)?), 911 ); 912 }; 913 }; 914 // When no `PinnedDrop` was specified, then we have to prevent implementing drop. 915 (drop_prevention: 916 @name($name:ident), 917 @impl_generics($($impl_generics:tt)*), 918 @ty_generics($($ty_generics:tt)*), 919 @where($($whr:tt)*), 920 @pinned_drop(), 921 ) => { 922 // We prevent this by creating a trait that will be implemented for all types implementing 923 // `Drop`. Additionally we will implement this trait for the struct leading to a conflict, 924 // if it also implements `Drop` 925 trait MustNotImplDrop {} 926 #[allow(drop_bounds)] 927 impl<T: ::core::ops::Drop> MustNotImplDrop for T {} 928 impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*> 929 where $($whr)* {} 930 // We also take care to prevent users from writing a useless `PinnedDrop` implementation. 931 // They might implement `PinnedDrop` correctly for the struct, but forget to give 932 // `PinnedDrop` as the parameter to `#[pin_data]`. 933 #[allow(non_camel_case_types)] 934 trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {} 935 impl<T: $crate::init::PinnedDrop> 936 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {} 937 impl<$($impl_generics)*> 938 UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*> 939 where $($whr)* {} 940 }; 941 // When `PinnedDrop` was specified we just implement `Drop` and delegate. 942 (drop_prevention: 943 @name($name:ident), 944 @impl_generics($($impl_generics:tt)*), 945 @ty_generics($($ty_generics:tt)*), 946 @where($($whr:tt)*), 947 @pinned_drop(PinnedDrop), 948 ) => { 949 impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*> 950 where $($whr)* 951 { 952 fn drop(&mut self) { 953 // SAFETY: Since this is a destructor, `self` will not move after this function 954 // terminates, since it is inaccessible. 955 let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) }; 956 // SAFETY: Since this is a drop function, we can create this token to call the 957 // pinned destructor of this type. 958 let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() }; 959 $crate::init::PinnedDrop::drop(pinned, token); 960 } 961 } 962 }; 963 // If some other parameter was specified, we emit a readable error. 964 (drop_prevention: 965 @name($name:ident), 966 @impl_generics($($impl_generics:tt)*), 967 @ty_generics($($ty_generics:tt)*), 968 @where($($whr:tt)*), 969 @pinned_drop($($rest:tt)*), 970 ) => { 971 compile_error!( 972 "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.", 973 stringify!($($rest)*), 974 ); 975 }; 976 (make_pin_data: 977 @pin_data($pin_data:ident), 978 @impl_generics($($impl_generics:tt)*), 979 @ty_generics($($ty_generics:tt)*), 980 @where($($whr:tt)*), 981 @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?), 982 @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?), 983 ) => { 984 // For every field, we create a projection function according to its projection type. If a 985 // field is structurally pinned, then it must be initialized via `PinInit`, if it is not 986 // structurally pinned, then it can be initialized via `Init`. 987 // 988 // The functions are `unsafe` to prevent accidentally calling them. 989 #[allow(dead_code)] 990 impl<$($impl_generics)*> $pin_data<$($ty_generics)*> 991 where $($whr)* 992 { 993 $( 994 $(#[$($p_attr)*])* 995 $pvis unsafe fn $p_field<E>( 996 self, 997 slot: *mut $p_type, 998 init: impl $crate::init::PinInit<$p_type, E>, 999 ) -> ::core::result::Result<(), E> { 1000 unsafe { $crate::init::PinInit::__pinned_init(init, slot) } 1001 } 1002 )* 1003 $( 1004 $(#[$($attr)*])* 1005 $fvis unsafe fn $field<E>( 1006 self, 1007 slot: *mut $type, 1008 init: impl $crate::init::Init<$type, E>, 1009 ) -> ::core::result::Result<(), E> { 1010 unsafe { $crate::init::Init::__init(init, slot) } 1011 } 1012 )* 1013 } 1014 }; 1015 } 1016 1017 /// The internal init macro. Do not call manually! 1018 /// 1019 /// This is called by the `{try_}{pin_}init!` macros with various inputs. 1020 /// 1021 /// This macro has multiple internal call configurations, these are always the very first ident: 1022 /// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros. 1023 /// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled. 1024 /// - `init_slot`: recursively creates the code that initializes all fields in `slot`. 1025 /// - `make_initializer`: recursively create the struct initializer that guarantees that every 1026 /// field has been initialized exactly once. 1027 #[doc(hidden)] 1028 #[macro_export] 1029 macro_rules! __init_internal { 1030 ( 1031 @this($($this:ident)?), 1032 @typ($t:path), 1033 @fields($($fields:tt)*), 1034 @error($err:ty), 1035 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1036 // case. 1037 @data($data:ident, $($use_data:ident)?), 1038 // `HasPinData` or `HasInitData`. 1039 @has_data($has_data:ident, $get_data:ident), 1040 // `pin_init_from_closure` or `init_from_closure`. 1041 @construct_closure($construct_closure:ident), 1042 @munch_fields(), 1043 ) => { 1044 $crate::__init_internal!(with_update_parsed: 1045 @this($($this)?), 1046 @typ($t), 1047 @fields($($fields)*), 1048 @error($err), 1049 @data($data, $($use_data)?), 1050 @has_data($has_data, $get_data), 1051 @construct_closure($construct_closure), 1052 @zeroed(), // Nothing means default behavior. 1053 ) 1054 }; 1055 ( 1056 @this($($this:ident)?), 1057 @typ($t:path), 1058 @fields($($fields:tt)*), 1059 @error($err:ty), 1060 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1061 // case. 1062 @data($data:ident, $($use_data:ident)?), 1063 // `HasPinData` or `HasInitData`. 1064 @has_data($has_data:ident, $get_data:ident), 1065 // `pin_init_from_closure` or `init_from_closure`. 1066 @construct_closure($construct_closure:ident), 1067 @munch_fields(..Zeroable::zeroed()), 1068 ) => { 1069 $crate::__init_internal!(with_update_parsed: 1070 @this($($this)?), 1071 @typ($t), 1072 @fields($($fields)*), 1073 @error($err), 1074 @data($data, $($use_data)?), 1075 @has_data($has_data, $get_data), 1076 @construct_closure($construct_closure), 1077 @zeroed(()), // `()` means zero all fields not mentioned. 1078 ) 1079 }; 1080 ( 1081 @this($($this:ident)?), 1082 @typ($t:path), 1083 @fields($($fields:tt)*), 1084 @error($err:ty), 1085 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1086 // case. 1087 @data($data:ident, $($use_data:ident)?), 1088 // `HasPinData` or `HasInitData`. 1089 @has_data($has_data:ident, $get_data:ident), 1090 // `pin_init_from_closure` or `init_from_closure`. 1091 @construct_closure($construct_closure:ident), 1092 @munch_fields($ignore:tt $($rest:tt)*), 1093 ) => { 1094 $crate::__init_internal!( 1095 @this($($this)?), 1096 @typ($t), 1097 @fields($($fields)*), 1098 @error($err), 1099 @data($data, $($use_data)?), 1100 @has_data($has_data, $get_data), 1101 @construct_closure($construct_closure), 1102 @munch_fields($($rest)*), 1103 ) 1104 }; 1105 (with_update_parsed: 1106 @this($($this:ident)?), 1107 @typ($t:path), 1108 @fields($($fields:tt)*), 1109 @error($err:ty), 1110 // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData` 1111 // case. 1112 @data($data:ident, $($use_data:ident)?), 1113 // `HasPinData` or `HasInitData`. 1114 @has_data($has_data:ident, $get_data:ident), 1115 // `pin_init_from_closure` or `init_from_closure`. 1116 @construct_closure($construct_closure:ident), 1117 @zeroed($($init_zeroed:expr)?), 1118 ) => {{ 1119 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return 1120 // type and shadow it later when we insert the arbitrary user code. That way there will be 1121 // no possibility of returning without `unsafe`. 1122 struct __InitOk; 1123 // Get the data about fields from the supplied type. 1124 let data = unsafe { 1125 use $crate::init::__internal::$has_data; 1126 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1127 // information that is associated to already parsed fragments, so a path fragment 1128 // cannot be used in this position. Doing the retokenization results in valid rust 1129 // code. 1130 ::kernel::macros::paste!($t::$get_data()) 1131 }; 1132 // Ensure that `data` really is of type `$data` and help with type inference: 1133 let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>( 1134 data, 1135 move |slot| { 1136 { 1137 // Shadow the structure so it cannot be used to return early. 1138 struct __InitOk; 1139 // If `$init_zeroed` is present we should zero the slot now and not emit an 1140 // error when fields are missing (since they will be zeroed). We also have to 1141 // check that the type actually implements `Zeroable`. 1142 $({ 1143 fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {} 1144 // Ensure that the struct is indeed `Zeroable`. 1145 assert_zeroable(slot); 1146 // SAFETY: The type implements `Zeroable` by the check above. 1147 unsafe { ::core::ptr::write_bytes(slot, 0, 1) }; 1148 $init_zeroed // This will be `()` if set. 1149 })? 1150 // Create the `this` so it can be referenced by the user inside of the 1151 // expressions creating the individual fields. 1152 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? 1153 // Initialize every field. 1154 $crate::__init_internal!(init_slot($($use_data)?): 1155 @data(data), 1156 @slot(slot), 1157 @guards(), 1158 @munch_fields($($fields)*,), 1159 ); 1160 // We use unreachable code to ensure that all fields have been mentioned exactly 1161 // once, this struct initializer will still be type-checked and complain with a 1162 // very natural error message if a field is forgotten/mentioned more than once. 1163 #[allow(unreachable_code, clippy::diverging_sub_expression)] 1164 let _ = || { 1165 $crate::__init_internal!(make_initializer: 1166 @slot(slot), 1167 @type_name($t), 1168 @munch_fields($($fields)*,), 1169 @acc(), 1170 ); 1171 }; 1172 } 1173 Ok(__InitOk) 1174 } 1175 ); 1176 let init = move |slot| -> ::core::result::Result<(), $err> { 1177 init(slot).map(|__InitOk| ()) 1178 }; 1179 let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) }; 1180 init 1181 }}; 1182 (init_slot($($use_data:ident)?): 1183 @data($data:ident), 1184 @slot($slot:ident), 1185 @guards($($guards:ident,)*), 1186 @munch_fields($(..Zeroable::zeroed())? $(,)?), 1187 ) => { 1188 // Endpoint of munching, no fields are left. If execution reaches this point, all fields 1189 // have been initialized. Therefore we can now dismiss the guards by forgetting them. 1190 $(::core::mem::forget($guards);)* 1191 }; 1192 (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields. 1193 @data($data:ident), 1194 @slot($slot:ident), 1195 @guards($($guards:ident,)*), 1196 // In-place initialization syntax. 1197 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1198 ) => { 1199 let init = $val; 1200 // Call the initializer. 1201 // 1202 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 1203 // return when an error/panic occurs. 1204 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. 1205 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? }; 1206 // Create the drop guard: 1207 // 1208 // We rely on macro hygiene to make it impossible for users to access this local variable. 1209 // We use `paste!` to create new hygiene for `$field`. 1210 ::kernel::macros::paste! { 1211 // SAFETY: We forget the guard later when initialization has succeeded. 1212 let [< __ $field _guard >] = unsafe { 1213 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1214 }; 1215 1216 $crate::__init_internal!(init_slot($use_data): 1217 @data($data), 1218 @slot($slot), 1219 @guards([< __ $field _guard >], $($guards,)*), 1220 @munch_fields($($rest)*), 1221 ); 1222 } 1223 }; 1224 (init_slot(): // No `use_data`, so we use `Init::__init` directly. 1225 @data($data:ident), 1226 @slot($slot:ident), 1227 @guards($($guards:ident,)*), 1228 // In-place initialization syntax. 1229 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1230 ) => { 1231 let init = $val; 1232 // Call the initializer. 1233 // 1234 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we 1235 // return when an error/panic occurs. 1236 unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? }; 1237 // Create the drop guard: 1238 // 1239 // We rely on macro hygiene to make it impossible for users to access this local variable. 1240 // We use `paste!` to create new hygiene for `$field`. 1241 ::kernel::macros::paste! { 1242 // SAFETY: We forget the guard later when initialization has succeeded. 1243 let [< __ $field _guard >] = unsafe { 1244 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1245 }; 1246 1247 $crate::__init_internal!(init_slot(): 1248 @data($data), 1249 @slot($slot), 1250 @guards([< __ $field _guard >], $($guards,)*), 1251 @munch_fields($($rest)*), 1252 ); 1253 } 1254 }; 1255 (init_slot($($use_data:ident)?): 1256 @data($data:ident), 1257 @slot($slot:ident), 1258 @guards($($guards:ident,)*), 1259 // Init by-value. 1260 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1261 ) => { 1262 { 1263 $(let $field = $val;)? 1264 // Initialize the field. 1265 // 1266 // SAFETY: The memory at `slot` is uninitialized. 1267 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; 1268 } 1269 // Create the drop guard: 1270 // 1271 // We rely on macro hygiene to make it impossible for users to access this local variable. 1272 // We use `paste!` to create new hygiene for `$field`. 1273 ::kernel::macros::paste! { 1274 // SAFETY: We forget the guard later when initialization has succeeded. 1275 let [< __ $field _guard >] = unsafe { 1276 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) 1277 }; 1278 1279 $crate::__init_internal!(init_slot($($use_data)?): 1280 @data($data), 1281 @slot($slot), 1282 @guards([< __ $field _guard >], $($guards,)*), 1283 @munch_fields($($rest)*), 1284 ); 1285 } 1286 }; 1287 (make_initializer: 1288 @slot($slot:ident), 1289 @type_name($t:path), 1290 @munch_fields(..Zeroable::zeroed() $(,)?), 1291 @acc($($acc:tt)*), 1292 ) => { 1293 // Endpoint, nothing more to munch, create the initializer. Since the users specified 1294 // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have 1295 // not been overwritten are thus zero and initialized. We still check that all fields are 1296 // actually accessible by using the struct update syntax ourselves. 1297 // We are inside of a closure that is never executed and thus we can abuse `slot` to 1298 // get the correct type inference here: 1299 #[allow(unused_assignments)] 1300 unsafe { 1301 let mut zeroed = ::core::mem::zeroed(); 1302 // We have to use type inference here to make zeroed have the correct type. This does 1303 // not get executed, so it has no effect. 1304 ::core::ptr::write($slot, zeroed); 1305 zeroed = ::core::mem::zeroed(); 1306 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1307 // information that is associated to already parsed fragments, so a path fragment 1308 // cannot be used in this position. Doing the retokenization results in valid rust 1309 // code. 1310 ::kernel::macros::paste!( 1311 ::core::ptr::write($slot, $t { 1312 $($acc)* 1313 ..zeroed 1314 }); 1315 ); 1316 } 1317 }; 1318 (make_initializer: 1319 @slot($slot:ident), 1320 @type_name($t:path), 1321 @munch_fields($(,)?), 1322 @acc($($acc:tt)*), 1323 ) => { 1324 // Endpoint, nothing more to munch, create the initializer. 1325 // Since we are in the closure that is never called, this will never get executed. 1326 // We abuse `slot` to get the correct type inference here: 1327 unsafe { 1328 // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal 1329 // information that is associated to already parsed fragments, so a path fragment 1330 // cannot be used in this position. Doing the retokenization results in valid rust 1331 // code. 1332 ::kernel::macros::paste!( 1333 ::core::ptr::write($slot, $t { 1334 $($acc)* 1335 }); 1336 ); 1337 } 1338 }; 1339 (make_initializer: 1340 @slot($slot:ident), 1341 @type_name($t:path), 1342 @munch_fields($field:ident <- $val:expr, $($rest:tt)*), 1343 @acc($($acc:tt)*), 1344 ) => { 1345 $crate::__init_internal!(make_initializer: 1346 @slot($slot), 1347 @type_name($t), 1348 @munch_fields($($rest)*), 1349 @acc($($acc)* $field: ::core::panic!(),), 1350 ); 1351 }; 1352 (make_initializer: 1353 @slot($slot:ident), 1354 @type_name($t:path), 1355 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), 1356 @acc($($acc:tt)*), 1357 ) => { 1358 $crate::__init_internal!(make_initializer: 1359 @slot($slot), 1360 @type_name($t), 1361 @munch_fields($($rest)*), 1362 @acc($($acc)* $field: ::core::panic!(),), 1363 ); 1364 }; 1365 } 1366 1367 #[doc(hidden)] 1368 #[macro_export] 1369 macro_rules! __derive_zeroable { 1370 (parse_input: 1371 @sig( 1372 $(#[$($struct_attr:tt)*])* 1373 $vis:vis struct $name:ident 1374 $(where $($whr:tt)*)? 1375 ), 1376 @impl_generics($($impl_generics:tt)*), 1377 @ty_generics($($ty_generics:tt)*), 1378 @body({ 1379 $( 1380 $(#[$($field_attr:tt)*])* 1381 $field:ident : $field_ty:ty 1382 ),* $(,)? 1383 }), 1384 ) => { 1385 // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero. 1386 #[automatically_derived] 1387 unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*> 1388 where 1389 $($($whr)*)? 1390 {} 1391 const _: () = { 1392 fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {} 1393 fn ensure_zeroable<$($impl_generics)*>() 1394 where $($($whr)*)? 1395 { 1396 $(assert_zeroable::<$field_ty>();)* 1397 } 1398 }; 1399 }; 1400 } 1401