1 //! The default matrix data storage allocator. 2 //! 3 //! This will use stack-allocated buffers for matrices with dimensions known at compile-time, and 4 //! heap-allocated buffers for matrices with at least one dimension unknown at compile-time. 5 6 use std::cmp; 7 use std::ptr; 8 9 #[cfg(all(feature = "alloc", not(feature = "std")))] 10 use alloc::vec::Vec; 11 12 use super::Const; 13 use crate::base::allocator::{Allocator, Reallocator}; 14 use crate::base::array_storage::ArrayStorage; 15 #[cfg(any(feature = "alloc", feature = "std"))] 16 use crate::base::dimension::Dynamic; 17 use crate::base::dimension::{Dim, DimName}; 18 use crate::base::storage::{RawStorage, RawStorageMut}; 19 #[cfg(any(feature = "std", feature = "alloc"))] 20 use crate::base::vec_storage::VecStorage; 21 use crate::base::Scalar; 22 use std::mem::{ManuallyDrop, MaybeUninit}; 23 24 /* 25 * 26 * Allocator. 27 * 28 */ 29 /// An allocator based on `GenericArray` and `VecStorage` for statically-sized and dynamically-sized 30 /// matrices respectively. 31 #[derive(Copy, Clone, Debug)] 32 pub struct DefaultAllocator; 33 34 // Static - Static 35 impl<T: Scalar, const R: usize, const C: usize> Allocator<T, Const<R>, Const<C>> 36 for DefaultAllocator 37 { 38 type Buffer = ArrayStorage<T, R, C>; 39 type BufferUninit = ArrayStorage<MaybeUninit<T>, R, C>; 40 41 #[inline(always)] allocate_uninit(_: Const<R>, _: Const<C>) -> ArrayStorage<MaybeUninit<T>, R, C>42 fn allocate_uninit(_: Const<R>, _: Const<C>) -> ArrayStorage<MaybeUninit<T>, R, C> { 43 // SAFETY: An uninitialized `[MaybeUninit<_>; _]` is valid. 44 let array: [[MaybeUninit<T>; R]; C] = unsafe { MaybeUninit::uninit().assume_init() }; 45 ArrayStorage(array) 46 } 47 48 #[inline(always)] assume_init(uninit: ArrayStorage<MaybeUninit<T>, R, C>) -> ArrayStorage<T, R, C>49 unsafe fn assume_init(uninit: ArrayStorage<MaybeUninit<T>, R, C>) -> ArrayStorage<T, R, C> { 50 // Safety: 51 // * The caller guarantees that all elements of the array are initialized 52 // * `MaybeUninit<T>` and T are guaranteed to have the same layout 53 // * `MaybeUninit` does not drop, so there are no double-frees 54 // And thus the conversion is safe 55 ArrayStorage((&uninit as *const _ as *const [_; C]).read()) 56 } 57 58 #[inline] allocate_from_iterator<I: IntoIterator<Item = T>>( nrows: Const<R>, ncols: Const<C>, iter: I, ) -> Self::Buffer59 fn allocate_from_iterator<I: IntoIterator<Item = T>>( 60 nrows: Const<R>, 61 ncols: Const<C>, 62 iter: I, 63 ) -> Self::Buffer { 64 let mut res = Self::allocate_uninit(nrows, ncols); 65 let mut count = 0; 66 67 // Safety: conversion to a slice is OK because the Buffer is known to be contiguous. 68 let res_slice = unsafe { res.as_mut_slice_unchecked() }; 69 for (res, e) in res_slice.iter_mut().zip(iter.into_iter()) { 70 *res = MaybeUninit::new(e); 71 count += 1; 72 } 73 74 assert!( 75 count == nrows.value() * ncols.value(), 76 "Matrix init. from iterator: iterator not long enough." 77 ); 78 79 // Safety: the assertion above made sure that the iterator 80 // yielded enough elements to initialize our matrix. 81 unsafe { <Self as Allocator<T, Const<R>, Const<C>>>::assume_init(res) } 82 } 83 } 84 85 // Dynamic - Static 86 // Dynamic - Dynamic 87 #[cfg(any(feature = "std", feature = "alloc"))] 88 impl<T: Scalar, C: Dim> Allocator<T, Dynamic, C> for DefaultAllocator { 89 type Buffer = VecStorage<T, Dynamic, C>; 90 type BufferUninit = VecStorage<MaybeUninit<T>, Dynamic, C>; 91 92 #[inline] allocate_uninit(nrows: Dynamic, ncols: C) -> VecStorage<MaybeUninit<T>, Dynamic, C>93 fn allocate_uninit(nrows: Dynamic, ncols: C) -> VecStorage<MaybeUninit<T>, Dynamic, C> { 94 let mut data = Vec::new(); 95 let length = nrows.value() * ncols.value(); 96 data.reserve_exact(length); 97 data.resize_with(length, MaybeUninit::uninit); 98 VecStorage::new(nrows, ncols, data) 99 } 100 101 #[inline] assume_init( uninit: VecStorage<MaybeUninit<T>, Dynamic, C>, ) -> VecStorage<T, Dynamic, C>102 unsafe fn assume_init( 103 uninit: VecStorage<MaybeUninit<T>, Dynamic, C>, 104 ) -> VecStorage<T, Dynamic, C> { 105 // Avoids a double-drop. 106 let (nrows, ncols) = uninit.shape(); 107 let vec: Vec<_> = uninit.into(); 108 let mut md = ManuallyDrop::new(vec); 109 110 // Safety: 111 // - MaybeUninit<T> has the same alignment and layout as T. 112 // - The length and capacity come from a valid vector. 113 let new_data = Vec::from_raw_parts(md.as_mut_ptr() as *mut _, md.len(), md.capacity()); 114 115 VecStorage::new(nrows, ncols, new_data) 116 } 117 118 #[inline] allocate_from_iterator<I: IntoIterator<Item = T>>( nrows: Dynamic, ncols: C, iter: I, ) -> Self::Buffer119 fn allocate_from_iterator<I: IntoIterator<Item = T>>( 120 nrows: Dynamic, 121 ncols: C, 122 iter: I, 123 ) -> Self::Buffer { 124 let it = iter.into_iter(); 125 let res: Vec<T> = it.collect(); 126 assert!(res.len() == nrows.value() * ncols.value(), 127 "Allocation from iterator error: the iterator did not yield the correct number of elements."); 128 129 VecStorage::new(nrows, ncols, res) 130 } 131 } 132 133 // Static - Dynamic 134 #[cfg(any(feature = "std", feature = "alloc"))] 135 impl<T: Scalar, R: DimName> Allocator<T, R, Dynamic> for DefaultAllocator { 136 type Buffer = VecStorage<T, R, Dynamic>; 137 type BufferUninit = VecStorage<MaybeUninit<T>, R, Dynamic>; 138 139 #[inline] allocate_uninit(nrows: R, ncols: Dynamic) -> VecStorage<MaybeUninit<T>, R, Dynamic>140 fn allocate_uninit(nrows: R, ncols: Dynamic) -> VecStorage<MaybeUninit<T>, R, Dynamic> { 141 let mut data = Vec::new(); 142 let length = nrows.value() * ncols.value(); 143 data.reserve_exact(length); 144 data.resize_with(length, MaybeUninit::uninit); 145 146 VecStorage::new(nrows, ncols, data) 147 } 148 149 #[inline] assume_init( uninit: VecStorage<MaybeUninit<T>, R, Dynamic>, ) -> VecStorage<T, R, Dynamic>150 unsafe fn assume_init( 151 uninit: VecStorage<MaybeUninit<T>, R, Dynamic>, 152 ) -> VecStorage<T, R, Dynamic> { 153 // Avoids a double-drop. 154 let (nrows, ncols) = uninit.shape(); 155 let vec: Vec<_> = uninit.into(); 156 let mut md = ManuallyDrop::new(vec); 157 158 // Safety: 159 // - MaybeUninit<T> has the same alignment and layout as T. 160 // - The length and capacity come from a valid vector. 161 let new_data = Vec::from_raw_parts(md.as_mut_ptr() as *mut _, md.len(), md.capacity()); 162 163 VecStorage::new(nrows, ncols, new_data) 164 } 165 166 #[inline] allocate_from_iterator<I: IntoIterator<Item = T>>( nrows: R, ncols: Dynamic, iter: I, ) -> Self::Buffer167 fn allocate_from_iterator<I: IntoIterator<Item = T>>( 168 nrows: R, 169 ncols: Dynamic, 170 iter: I, 171 ) -> Self::Buffer { 172 let it = iter.into_iter(); 173 let res: Vec<T> = it.collect(); 174 assert!(res.len() == nrows.value() * ncols.value(), 175 "Allocation from iterator error: the iterator did not yield the correct number of elements."); 176 177 VecStorage::new(nrows, ncols, res) 178 } 179 } 180 181 /* 182 * 183 * Reallocator. 184 * 185 */ 186 // Anything -> Static × Static 187 impl<T: Scalar, RFrom, CFrom, const RTO: usize, const CTO: usize> 188 Reallocator<T, RFrom, CFrom, Const<RTO>, Const<CTO>> for DefaultAllocator 189 where 190 RFrom: Dim, 191 CFrom: Dim, 192 Self: Allocator<T, RFrom, CFrom>, 193 { 194 #[inline] reallocate_copy( rto: Const<RTO>, cto: Const<CTO>, mut buf: <Self as Allocator<T, RFrom, CFrom>>::Buffer, ) -> ArrayStorage<MaybeUninit<T>, RTO, CTO>195 unsafe fn reallocate_copy( 196 rto: Const<RTO>, 197 cto: Const<CTO>, 198 mut buf: <Self as Allocator<T, RFrom, CFrom>>::Buffer, 199 ) -> ArrayStorage<MaybeUninit<T>, RTO, CTO> { 200 let mut res = <Self as Allocator<T, Const<RTO>, Const<CTO>>>::allocate_uninit(rto, cto); 201 202 let (rfrom, cfrom) = buf.shape(); 203 204 let len_from = rfrom.value() * cfrom.value(); 205 let len_to = rto.value() * cto.value(); 206 let len_copied = cmp::min(len_from, len_to); 207 ptr::copy_nonoverlapping(buf.ptr(), res.ptr_mut() as *mut T, len_copied); 208 209 // Safety: 210 // - We don’t care about dropping elements because the caller is responsible for dropping things. 211 // - We forget `buf` so that we don’t drop the other elements. 212 std::mem::forget(buf); 213 214 res 215 } 216 } 217 218 // Static × Static -> Dynamic × Any 219 #[cfg(any(feature = "std", feature = "alloc"))] 220 impl<T: Scalar, CTo, const RFROM: usize, const CFROM: usize> 221 Reallocator<T, Const<RFROM>, Const<CFROM>, Dynamic, CTo> for DefaultAllocator 222 where 223 CTo: Dim, 224 { 225 #[inline] reallocate_copy( rto: Dynamic, cto: CTo, mut buf: ArrayStorage<T, RFROM, CFROM>, ) -> VecStorage<MaybeUninit<T>, Dynamic, CTo>226 unsafe fn reallocate_copy( 227 rto: Dynamic, 228 cto: CTo, 229 mut buf: ArrayStorage<T, RFROM, CFROM>, 230 ) -> VecStorage<MaybeUninit<T>, Dynamic, CTo> { 231 let mut res = <Self as Allocator<T, Dynamic, CTo>>::allocate_uninit(rto, cto); 232 233 let (rfrom, cfrom) = buf.shape(); 234 235 let len_from = rfrom.value() * cfrom.value(); 236 let len_to = rto.value() * cto.value(); 237 let len_copied = cmp::min(len_from, len_to); 238 ptr::copy_nonoverlapping(buf.ptr(), res.ptr_mut() as *mut T, len_copied); 239 240 // Safety: 241 // - We don’t care about dropping elements because the caller is responsible for dropping things. 242 // - We forget `buf` so that we don’t drop the other elements. 243 std::mem::forget(buf); 244 245 res 246 } 247 } 248 249 // Static × Static -> Static × Dynamic 250 #[cfg(any(feature = "std", feature = "alloc"))] 251 impl<T: Scalar, RTo, const RFROM: usize, const CFROM: usize> 252 Reallocator<T, Const<RFROM>, Const<CFROM>, RTo, Dynamic> for DefaultAllocator 253 where 254 RTo: DimName, 255 { 256 #[inline] reallocate_copy( rto: RTo, cto: Dynamic, mut buf: ArrayStorage<T, RFROM, CFROM>, ) -> VecStorage<MaybeUninit<T>, RTo, Dynamic>257 unsafe fn reallocate_copy( 258 rto: RTo, 259 cto: Dynamic, 260 mut buf: ArrayStorage<T, RFROM, CFROM>, 261 ) -> VecStorage<MaybeUninit<T>, RTo, Dynamic> { 262 let mut res = <Self as Allocator<T, RTo, Dynamic>>::allocate_uninit(rto, cto); 263 264 let (rfrom, cfrom) = buf.shape(); 265 266 let len_from = rfrom.value() * cfrom.value(); 267 let len_to = rto.value() * cto.value(); 268 let len_copied = cmp::min(len_from, len_to); 269 ptr::copy_nonoverlapping(buf.ptr(), res.ptr_mut() as *mut T, len_copied); 270 271 // Safety: 272 // - We don’t care about dropping elements because the caller is responsible for dropping things. 273 // - We forget `buf` so that we don’t drop the other elements. 274 std::mem::forget(buf); 275 276 res 277 } 278 } 279 280 // All conversion from a dynamic buffer to a dynamic buffer. 281 #[cfg(any(feature = "std", feature = "alloc"))] 282 impl<T: Scalar, CFrom: Dim, CTo: Dim> Reallocator<T, Dynamic, CFrom, Dynamic, CTo> 283 for DefaultAllocator 284 { 285 #[inline] reallocate_copy( rto: Dynamic, cto: CTo, buf: VecStorage<T, Dynamic, CFrom>, ) -> VecStorage<MaybeUninit<T>, Dynamic, CTo>286 unsafe fn reallocate_copy( 287 rto: Dynamic, 288 cto: CTo, 289 buf: VecStorage<T, Dynamic, CFrom>, 290 ) -> VecStorage<MaybeUninit<T>, Dynamic, CTo> { 291 let new_buf = buf.resize(rto.value() * cto.value()); 292 VecStorage::new(rto, cto, new_buf) 293 } 294 } 295 296 #[cfg(any(feature = "std", feature = "alloc"))] 297 impl<T: Scalar, CFrom: Dim, RTo: DimName> Reallocator<T, Dynamic, CFrom, RTo, Dynamic> 298 for DefaultAllocator 299 { 300 #[inline] reallocate_copy( rto: RTo, cto: Dynamic, buf: VecStorage<T, Dynamic, CFrom>, ) -> VecStorage<MaybeUninit<T>, RTo, Dynamic>301 unsafe fn reallocate_copy( 302 rto: RTo, 303 cto: Dynamic, 304 buf: VecStorage<T, Dynamic, CFrom>, 305 ) -> VecStorage<MaybeUninit<T>, RTo, Dynamic> { 306 let new_buf = buf.resize(rto.value() * cto.value()); 307 VecStorage::new(rto, cto, new_buf) 308 } 309 } 310 311 #[cfg(any(feature = "std", feature = "alloc"))] 312 impl<T: Scalar, RFrom: DimName, CTo: Dim> Reallocator<T, RFrom, Dynamic, Dynamic, CTo> 313 for DefaultAllocator 314 { 315 #[inline] reallocate_copy( rto: Dynamic, cto: CTo, buf: VecStorage<T, RFrom, Dynamic>, ) -> VecStorage<MaybeUninit<T>, Dynamic, CTo>316 unsafe fn reallocate_copy( 317 rto: Dynamic, 318 cto: CTo, 319 buf: VecStorage<T, RFrom, Dynamic>, 320 ) -> VecStorage<MaybeUninit<T>, Dynamic, CTo> { 321 let new_buf = buf.resize(rto.value() * cto.value()); 322 VecStorage::new(rto, cto, new_buf) 323 } 324 } 325 326 #[cfg(any(feature = "std", feature = "alloc"))] 327 impl<T: Scalar, RFrom: DimName, RTo: DimName> Reallocator<T, RFrom, Dynamic, RTo, Dynamic> 328 for DefaultAllocator 329 { 330 #[inline] reallocate_copy( rto: RTo, cto: Dynamic, buf: VecStorage<T, RFrom, Dynamic>, ) -> VecStorage<MaybeUninit<T>, RTo, Dynamic>331 unsafe fn reallocate_copy( 332 rto: RTo, 333 cto: Dynamic, 334 buf: VecStorage<T, RFrom, Dynamic>, 335 ) -> VecStorage<MaybeUninit<T>, RTo, Dynamic> { 336 let new_buf = buf.resize(rto.value() * cto.value()); 337 VecStorage::new(rto, cto, new_buf) 338 } 339 } 340