1 //============================================================================
2 // Copyright (c) Kitware, Inc.
3 // All rights reserved.
4 // See LICENSE.txt for details.
5 //
6 // This software is distributed WITHOUT ANY WARRANTY; without even
7 // the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
8 // PURPOSE. See the above copyright notice for more information.
9 //============================================================================
10
11 #include <vtkm/internal/Assume.h>
12
13 #include <vtkm/cont/ErrorBadAllocation.h>
14 #include <vtkm/cont/ErrorBadDevice.h>
15 #include <vtkm/cont/ErrorBadType.h>
16 #include <vtkm/cont/RuntimeDeviceInformation.h>
17
18 #include <vtkm/cont/internal/Buffer.h>
19 #include <vtkm/cont/internal/DeviceAdapterMemoryManager.h>
20
21 #include <condition_variable>
22 #include <cstring>
23 #include <deque>
24 #include <map>
25 #include <memory>
26
27 namespace vtkm
28 {
29 namespace internal
30 {
31
NumberOfValuesToNumberOfBytes(vtkm::Id numValues,std::size_t typeSize)32 vtkm::BufferSizeType NumberOfValuesToNumberOfBytes(vtkm::Id numValues, std::size_t typeSize)
33 {
34 VTKM_ASSERT(numValues >= 0);
35
36 if (numValues > (std::numeric_limits<vtkm::BufferSizeType>::max() /
37 static_cast<vtkm::BufferSizeType>(typeSize)))
38 {
39 throw vtkm::cont::ErrorBadAllocation("Asking for a buffer too big to represent.");
40 }
41
42 return numValues * static_cast<vtkm::BufferSizeType>(typeSize);
43 }
44 }
45 } // namespace vtkm::internal
46
47 namespace
48 {
49
50 using LockType = std::unique_lock<std::mutex>;
51
52 struct BufferState
53 {
54 vtkm::cont::internal::BufferInfo Info;
55 bool Pinned = false;
56 bool UpToDate = false;
57
58 BufferState() = default;
BufferState__anon8994aed10111::BufferState59 BufferState(const vtkm::cont::internal::BufferInfo& info,
60 bool pinned = false,
61 bool upToDate = true)
62 : Info(info)
63 , Pinned(pinned)
64 , UpToDate(upToDate)
65 {
66 }
67
68 // Automatically convert to BufferInfo
operator vtkm::cont::internal::BufferInfo&__anon8994aed10111::BufferState69 operator vtkm::cont::internal::BufferInfo&() { return this->Info; }
operator const vtkm::cont::internal::BufferInfo&__anon8994aed10111::BufferState70 operator const vtkm::cont::internal::BufferInfo&() const { return this->Info; }
71
72 // Pass through common BufferInfo methods.
73
GetPointer__anon8994aed10111::BufferState74 void* GetPointer() const { return this->Info.GetPointer(); }
75
GetSize__anon8994aed10111::BufferState76 vtkm::BufferSizeType GetSize() const { return this->Info.GetSize(); }
77
GetDevice__anon8994aed10111::BufferState78 vtkm::cont::DeviceAdapterId GetDevice() const { return this->Info.GetDevice(); }
79
Reallocate__anon8994aed10111::BufferState80 void Reallocate(vtkm::BufferSizeType newSize)
81 {
82 if (this->Info.GetSize() != newSize)
83 {
84 if (this->Pinned)
85 {
86 throw vtkm::cont::ErrorBadAllocation("Attempted to reallocate a pinned buffer.");
87 }
88 this->Info.Reallocate(newSize);
89 }
90 }
91
92 /// Releases the buffer. If the memory is not pinned, it is deleted. In any case, it is
93 /// marked as no longer up to date.
Release__anon8994aed10111::BufferState94 void Release()
95 {
96 if (!this->Pinned)
97 {
98 this->Info = vtkm::cont::internal::BufferInfo{};
99 }
100 this->UpToDate = false;
101 }
102 };
103
104 struct MetaDataManager
105 {
106 void* Data = nullptr;
107 std::string Type;
108 vtkm::cont::internal::detail::DeleterType* Deleter = nullptr;
109 vtkm::cont::internal::detail::CopierType* Copier = nullptr;
110
111 MetaDataManager() = default;
112
~MetaDataManager__anon8994aed10111::MetaDataManager113 ~MetaDataManager()
114 {
115 if (this->Data != nullptr)
116 {
117 VTKM_ASSERT(this->Deleter != nullptr);
118 this->Deleter(this->Data);
119 this->Data = nullptr;
120 }
121 }
122
123 // We don't know how much information is the metadata, and copying it could be expensive.
124 // Thus, we want to be intentional about copying the metadata only for deep copies.
125 MetaDataManager(const MetaDataManager& src) = delete;
126 MetaDataManager& operator=(const MetaDataManager& src) = delete;
127
Initialize__anon8994aed10111::MetaDataManager128 void Initialize(void* data,
129 const std::string& type,
130 vtkm::cont::internal::detail::DeleterType* deleter,
131 vtkm::cont::internal::detail::CopierType* copier)
132 {
133 Data = data;
134 Type = type;
135 Deleter = deleter;
136 Copier = copier;
137 }
138
DeepCopyFrom__anon8994aed10111::MetaDataManager139 void DeepCopyFrom(const MetaDataManager& src)
140 {
141 if (this->Data != nullptr)
142 {
143 VTKM_ASSERT(this->Deleter != nullptr);
144 this->Deleter(this->Data);
145 this->Data = nullptr;
146 this->Type = "";
147 }
148 if (src.Data != nullptr)
149 {
150 VTKM_ASSERT(src.Copier);
151 VTKM_ASSERT(src.Deleter);
152 this->Data = src.Copier(src.Data);
153 this->Type = src.Type;
154 this->Deleter = src.Deleter;
155 this->Copier = src.Copier;
156 }
157 }
158 };
159
160 } // anonymous namespace
161
162 namespace vtkm
163 {
164 namespace cont
165 {
166 namespace internal
167 {
168
169 class Buffer::InternalsStruct
170 {
171 public:
172 using DeviceBufferMap = std::map<vtkm::cont::DeviceAdapterId, BufferState>;
173
174 private:
175 vtkm::cont::Token::ReferenceCount ReadCount = 0;
176 vtkm::cont::Token::ReferenceCount WriteCount = 0;
177
178 std::deque<vtkm::cont::Token::Reference> Queue;
179
CheckLock(const LockType & lock) const180 VTKM_CONT void CheckLock(const LockType& lock) const
181 {
182 VTKM_ASSERT((lock.mutex() == &this->Mutex) && (lock.owns_lock()));
183 }
184
185 // If this number disagrees with the size of the buffers, then they should be resized and
186 // data preserved.
187 vtkm::BufferSizeType NumberOfBytes = 0;
188
189 DeviceBufferMap DeviceBuffers;
190 BufferState HostBuffer;
191
192 public:
193 std::mutex Mutex;
194 std::condition_variable ConditionVariable;
195
196 MetaDataManager MetaData;
197
GetLock()198 LockType GetLock() { return LockType(this->Mutex); }
199
GetQueue(const LockType & lock)200 VTKM_CONT std::deque<vtkm::cont::Token::Reference>& GetQueue(const LockType& lock)
201 {
202 this->CheckLock(lock);
203 return this->Queue;
204 }
205
GetReadCount(const LockType & lock)206 VTKM_CONT vtkm::cont::Token::ReferenceCount* GetReadCount(const LockType& lock)
207 {
208 this->CheckLock(lock);
209 return &this->ReadCount;
210 }
GetWriteCount(const LockType & lock)211 VTKM_CONT vtkm::cont::Token::ReferenceCount* GetWriteCount(const LockType& lock)
212 {
213 this->CheckLock(lock);
214 return &this->WriteCount;
215 }
216
GetDeviceBuffers(const LockType & lock)217 VTKM_CONT DeviceBufferMap& GetDeviceBuffers(const LockType& lock)
218 {
219 this->CheckLock(lock);
220 return this->DeviceBuffers;
221 }
222
GetHostBuffer(const LockType & lock)223 VTKM_CONT BufferState& GetHostBuffer(const LockType& lock)
224 {
225 this->CheckLock(lock);
226 return this->HostBuffer;
227 }
228
GetNumberOfBytes(const LockType & lock)229 VTKM_CONT vtkm::BufferSizeType GetNumberOfBytes(const LockType& lock)
230 {
231 this->CheckLock(lock);
232 return this->NumberOfBytes;
233 }
SetNumberOfBytes(const LockType & lock,vtkm::BufferSizeType numberOfBytes)234 VTKM_CONT void SetNumberOfBytes(const LockType& lock, vtkm::BufferSizeType numberOfBytes)
235 {
236 this->CheckLock(lock);
237 this->NumberOfBytes = numberOfBytes;
238 }
239 };
240
241 namespace detail
242 {
243
244 struct VTKM_NEVER_EXPORT BufferHelper
245 {
246 enum struct AccessMode
247 {
248 READ,
249 WRITE
250 };
251
Enqueuevtkm::cont::internal::detail::BufferHelper252 static void Enqueue(const std::shared_ptr<Buffer::InternalsStruct>& internals,
253 const LockType& lock,
254 const vtkm::cont::Token& token)
255 {
256 if (token.IsAttached(internals->GetWriteCount(lock)) ||
257 token.IsAttached(internals->GetReadCount(lock)))
258 {
259 // Do not need to enqueue if we are already attached.
260 return;
261 }
262
263 auto& queue = internals->GetQueue(lock);
264 if (std::find(queue.begin(), queue.end(), token.GetReference()) != queue.end())
265 {
266 // This token is already in the queue.
267 return;
268 }
269
270 queue.push_back(token.GetReference());
271 }
272
CanReadvtkm::cont::internal::detail::BufferHelper273 static bool CanRead(const std::shared_ptr<Buffer::InternalsStruct>& internals,
274 const LockType& lock,
275 const vtkm::cont::Token& token)
276 {
277 // If the token is already attached to this array, then we allow reading.
278 if (token.IsAttached(internals->GetWriteCount(lock)) ||
279 token.IsAttached(internals->GetReadCount(lock)))
280 {
281 return true;
282 }
283
284 // If there is anyone else waiting at the top of the queue, we cannot access this array.
285 auto& queue = internals->GetQueue(lock);
286 if (!queue.empty() && (queue.front() != token))
287 {
288 return false;
289 }
290
291 // No one else is waiting, so we can read the buffer as long as no one else is writing.
292 return (*internals->GetWriteCount(lock) < 1);
293 }
294
CanWritevtkm::cont::internal::detail::BufferHelper295 static bool CanWrite(const std::shared_ptr<Buffer::InternalsStruct>& internals,
296 const LockType& lock,
297 const vtkm::cont::Token& token)
298 {
299 // If the token is already attached to this array, then we allow writing.
300 if (token.IsAttached(internals->GetWriteCount(lock)) ||
301 token.IsAttached(internals->GetReadCount(lock)))
302 {
303 return true;
304 }
305
306 // If there is anyone else waiting at the top of the queue, we cannot access this array.
307 auto& queue = internals->GetQueue(lock);
308 if (!queue.empty() && (queue.front() != token))
309 {
310 return false;
311 }
312
313 // No one else is waiting, so we can write the buffer as long as no one else is reading
314 // or writing.
315 return ((*internals->GetWriteCount(lock) < 1) && (*internals->GetReadCount(lock) < 1));
316 }
317
WaitToReadvtkm::cont::internal::detail::BufferHelper318 static void WaitToRead(const std::shared_ptr<Buffer::InternalsStruct>& internals,
319 LockType& lock,
320 vtkm::cont::Token& token)
321 {
322 Enqueue(internals, lock, token);
323
324 // Note that if you deadlocked here, that means that you are trying to do a read operation on an
325 // array where an object is writing to it. This could happen on the same thread. For example, if
326 // you call `WritePortal()` then no other operation that can result in reading or writing
327 // data in the array can happen while the resulting portal is still in scope.
328 internals->ConditionVariable.wait(
329 lock, [&lock, &token, internals] { return CanRead(internals, lock, token); });
330
331 token.Attach(internals, internals->GetReadCount(lock), lock, &internals->ConditionVariable);
332
333 // We successfully attached the token. Pop it off the queue.
334 auto& queue = internals->GetQueue(lock);
335 if (!queue.empty() && queue.front() == token)
336 {
337 queue.pop_front();
338 }
339 }
340
WaitToWritevtkm::cont::internal::detail::BufferHelper341 static void WaitToWrite(const std::shared_ptr<Buffer::InternalsStruct>& internals,
342 LockType& lock,
343 vtkm::cont::Token& token)
344 {
345 Enqueue(internals, lock, token);
346
347 // Note that if you deadlocked here, that means that you are trying to do a write operation on
348 // an array where an object is reading or writing to it. This could happen on the same thread.
349 // For example, if you call `WritePortal()` then no other operation that can result in reading
350 // or writing data in the array can happen while the resulting portal is still in scope.
351 internals->ConditionVariable.wait(
352 lock, [&lock, &token, internals] { return CanWrite(internals, lock, token); });
353
354 token.Attach(internals, internals->GetWriteCount(lock), lock, &internals->ConditionVariable);
355
356 // We successfully attached the token. Pop it off the queue.
357 auto& queue = internals->GetQueue(lock);
358 if (!queue.empty() && queue.front() == token)
359 {
360 queue.pop_front();
361 }
362 }
363
Waitvtkm::cont::internal::detail::BufferHelper364 static void Wait(const std::shared_ptr<Buffer::InternalsStruct>& internals,
365 LockType& lock,
366 vtkm::cont::Token& token,
367 AccessMode accessMode)
368 {
369 switch (accessMode)
370 {
371 case AccessMode::READ:
372 WaitToRead(internals, lock, token);
373 break;
374 case AccessMode::WRITE:
375 WaitToWrite(internals, lock, token);
376 break;
377 }
378 }
379
SetNumberOfBytesvtkm::cont::internal::detail::BufferHelper380 static void SetNumberOfBytes(const std::shared_ptr<Buffer::InternalsStruct>& internals,
381 std::unique_lock<std::mutex>& lock,
382 vtkm::BufferSizeType numberOfBytes,
383 vtkm::CopyFlag preserve,
384 vtkm::cont::Token& token)
385 {
386 VTKM_ASSUME(numberOfBytes >= 0);
387
388 if (internals->GetNumberOfBytes(lock) == numberOfBytes)
389 {
390 // Allocation has not changed. Just return.
391 // Note, if you set the size to the old size and then try to get the buffer on a different
392 // place, a copy might happen.
393 return;
394 }
395
396 // We are altering the array, so make sure we can write to it.
397 BufferHelper::WaitToWrite(internals, lock, token);
398
399 internals->SetNumberOfBytes(lock, numberOfBytes);
400 if ((preserve == vtkm::CopyFlag::Off) || (numberOfBytes == 0))
401 {
402 // No longer need these buffers. Just release them.
403 internals->GetHostBuffer(lock).Release();
404 for (auto&& deviceBuffer : internals->GetDeviceBuffers(lock))
405 {
406 deviceBuffer.second.Release();
407 }
408 }
409 else
410 {
411 // Do nothing (other than resetting numberOfBytes). Buffers will get resized when you get the
412 // pointer.
413 }
414 }
415
AllocateOnHostvtkm::cont::internal::detail::BufferHelper416 static void AllocateOnHost(const std::shared_ptr<Buffer::InternalsStruct>& internals,
417 std::unique_lock<std::mutex>& lock,
418 vtkm::cont::Token& token,
419 AccessMode accessMode)
420 {
421 Wait(internals, lock, token, accessMode);
422 BufferState& hostBuffer = internals->GetHostBuffer(lock);
423 vtkm::BufferSizeType targetSize = internals->GetNumberOfBytes(lock);
424 if (hostBuffer.UpToDate)
425 {
426 // Buffer already exists on the host. Make sure it is the right size.
427 if (hostBuffer.GetSize() != targetSize)
428 {
429 hostBuffer.Reallocate(targetSize);
430 }
431 return;
432 }
433
434 // Buffer does not exist on host. See if we can find data on a device.
435 for (auto&& deviceBuffer : internals->GetDeviceBuffers(lock))
436 {
437 if (!deviceBuffer.second.UpToDate)
438 {
439 continue;
440 }
441
442 vtkm::cont::internal::DeviceAdapterMemoryManagerBase& memoryManager =
443 vtkm::cont::RuntimeDeviceInformation().GetMemoryManager(deviceBuffer.first);
444
445 if (deviceBuffer.second.GetSize() > targetSize)
446 {
447 // Device buffer too large. Resize.
448 deviceBuffer.second.Reallocate(targetSize);
449 }
450
451 if (!hostBuffer.Pinned)
452 {
453 hostBuffer = memoryManager.CopyDeviceToHost(deviceBuffer.second);
454 }
455 else
456 {
457 hostBuffer.Reallocate(targetSize);
458 memoryManager.CopyDeviceToHost(deviceBuffer.second, hostBuffer);
459 }
460
461 if (hostBuffer.GetSize() != targetSize)
462 {
463 hostBuffer.Reallocate(targetSize);
464 }
465
466 hostBuffer.UpToDate = true;
467
468 return;
469 }
470
471 // Buffer not up to date on host or any device, so just allocate a buffer.
472 if (!hostBuffer.Pinned)
473 {
474 hostBuffer = vtkm::cont::internal::AllocateOnHost(targetSize);
475 }
476 else
477 {
478 hostBuffer.Reallocate(targetSize);
479 hostBuffer.UpToDate = true;
480 }
481 }
482
AllocateOnDevicevtkm::cont::internal::detail::BufferHelper483 static void AllocateOnDevice(const std::shared_ptr<Buffer::InternalsStruct>& internals,
484 std::unique_lock<std::mutex>& lock,
485 vtkm::cont::Token& token,
486 vtkm::cont::DeviceAdapterId device,
487 AccessMode accessMode)
488 {
489 Wait(internals, lock, token, accessMode);
490 Buffer::InternalsStruct::DeviceBufferMap& deviceBuffers = internals->GetDeviceBuffers(lock);
491 vtkm::BufferSizeType targetSize = internals->GetNumberOfBytes(lock);
492 vtkm::cont::internal::DeviceAdapterMemoryManagerBase& memoryManager =
493 vtkm::cont::RuntimeDeviceInformation().GetMemoryManager(device);
494
495 if (deviceBuffers[device].UpToDate)
496 {
497 // Buffer already exists on the device. Make sure it is the right size.
498 if (deviceBuffers[device].GetSize() != targetSize)
499 {
500 deviceBuffers[device].Reallocate(targetSize);
501 }
502 VTKM_ASSERT(deviceBuffers[device].GetSize() == targetSize);
503 return;
504 }
505
506 // Buffer does not exist on device. Check to see if it is on another device but not the host.
507 // We currently do not support device-to-device transfers, so the data has to go to the
508 // host first.
509 if (!internals->GetHostBuffer(lock).UpToDate)
510 {
511 for (auto&& deviceBuffer : deviceBuffers)
512 {
513 if (deviceBuffer.second.UpToDate)
514 {
515 // Copy data to host.
516 AllocateOnHost(internals, lock, token, accessMode);
517 break;
518 }
519 }
520 }
521
522 // If the buffer is now on the host, copy it to the device.
523 BufferState& hostBuffer = internals->GetHostBuffer(lock);
524 if (hostBuffer.UpToDate)
525 {
526 if (hostBuffer.GetSize() > targetSize)
527 {
528 // Host buffer too large. Resize.
529 hostBuffer.Reallocate(targetSize);
530 }
531
532 if (!deviceBuffers[device].Pinned)
533 {
534 deviceBuffers[device] = memoryManager.CopyHostToDevice(hostBuffer);
535 }
536 else
537 {
538 deviceBuffers[device].Reallocate(targetSize);
539 memoryManager.CopyHostToDevice(hostBuffer, deviceBuffers[device]);
540 }
541
542 if (deviceBuffers[device].GetSize() != targetSize)
543 {
544 deviceBuffers[device].Reallocate(targetSize);
545 }
546 VTKM_ASSERT(deviceBuffers[device].GetSize() == targetSize);
547
548 deviceBuffers[device].UpToDate = true;
549
550 return;
551 }
552
553 // Buffer not up to date anywhere, so just allocate a buffer.
554 if (!deviceBuffers[device].Pinned)
555 {
556 deviceBuffers[device] = memoryManager.Allocate(targetSize);
557 }
558 else
559 {
560 deviceBuffers[device].Reallocate(targetSize);
561 deviceBuffers[device].UpToDate = true;
562 }
563 }
564
CopyOnHostvtkm::cont::internal::detail::BufferHelper565 static void CopyOnHost(
566 const std::shared_ptr<vtkm::cont::internal::Buffer::InternalsStruct>& srcInternals,
567 LockType& srcLock,
568 const std::shared_ptr<vtkm::cont::internal::Buffer::InternalsStruct>& destInternals,
569 LockType& destLock,
570 vtkm::cont::Token& token)
571 {
572 WaitToRead(srcInternals, srcLock, token);
573 WaitToWrite(destInternals, destLock, token);
574
575 vtkm::BufferSizeType size = srcInternals->GetNumberOfBytes(srcLock);
576
577 // Any current buffers in destination can be (and should be) deleted.
578 // Do this before allocating on the host to avoid unnecessary data copies.
579 for (auto&& deviceBuffer : destInternals->GetDeviceBuffers(destLock))
580 {
581 deviceBuffer.second.Release();
582 }
583
584 destInternals->SetNumberOfBytes(destLock, size);
585 AllocateOnHost(destInternals, destLock, token, AccessMode::WRITE);
586
587 AllocateOnHost(srcInternals, srcLock, token, AccessMode::READ);
588
589 std::memcpy(destInternals->GetHostBuffer(destLock).GetPointer(),
590 srcInternals->GetHostBuffer(srcLock).GetPointer(),
591 static_cast<std::size_t>(size));
592
593 destInternals->MetaData.DeepCopyFrom(srcInternals->MetaData);
594 }
595
CopyOnDevicevtkm::cont::internal::detail::BufferHelper596 static void CopyOnDevice(
597 vtkm::cont::DeviceAdapterId device,
598 const std::shared_ptr<vtkm::cont::internal::Buffer::InternalsStruct>& srcInternals,
599 LockType& srcLock,
600 const std::shared_ptr<vtkm::cont::internal::Buffer::InternalsStruct>& destInternals,
601 LockType& destLock,
602 vtkm::cont::Token& token)
603 {
604 WaitToRead(srcInternals, srcLock, token);
605 WaitToWrite(destInternals, destLock, token);
606
607 // Any current buffers in destination can be (and should be) deleted.
608 // Do this before allocating on the host to avoid unnecessary data copies.
609 vtkm::cont::internal::Buffer::InternalsStruct::DeviceBufferMap& destDeviceBuffers =
610 destInternals->GetDeviceBuffers(destLock);
611 destInternals->GetHostBuffer(destLock).Release();
612 for (auto&& deviceBuffer : destDeviceBuffers)
613 {
614 deviceBuffer.second.Release();
615 }
616
617 // Do the copy
618 vtkm::cont::internal::DeviceAdapterMemoryManagerBase& memoryManager =
619 vtkm::cont::RuntimeDeviceInformation().GetMemoryManager(device);
620
621 if (!destDeviceBuffers[device].Pinned)
622 {
623 destDeviceBuffers[device] =
624 memoryManager.CopyDeviceToDevice(srcInternals->GetDeviceBuffers(srcLock)[device]);
625 }
626 else
627 {
628 memoryManager.CopyDeviceToDevice(srcInternals->GetDeviceBuffers(srcLock)[device],
629 destDeviceBuffers[device]);
630 destDeviceBuffers[device].UpToDate = true;
631 }
632
633 destInternals->SetNumberOfBytes(destLock, srcInternals->GetNumberOfBytes(srcLock));
634
635 destInternals->MetaData.DeepCopyFrom(srcInternals->MetaData);
636 }
637 };
638
639 } // namespace detail
640
Buffer()641 Buffer::Buffer()
642 : Internals(new InternalsStruct)
643 {
644 }
645
646 // Defined to prevent issues with CUDA
Buffer(const Buffer & src)647 Buffer::Buffer(const Buffer& src)
648 : Internals(src.Internals)
649 {
650 }
651
652 // Defined to prevent issues with CUDA
Buffer(Buffer && src)653 Buffer::Buffer(Buffer&& src) noexcept
654 : Internals(std::move(src.Internals))
655 {
656 }
657
658 // Defined to prevent issues with CUDA
~Buffer()659 Buffer::~Buffer() {}
660
661 // Defined to prevent issues with CUDA
operator =(const Buffer & src)662 Buffer& Buffer::operator=(const Buffer& src)
663 {
664 this->Internals = src.Internals;
665 return *this;
666 }
667
668 // Defined to prevent issues with CUDA
operator =(Buffer && src)669 Buffer& Buffer::operator=(Buffer&& src) noexcept
670 {
671 this->Internals = std::move(src.Internals);
672 return *this;
673 }
674
GetNumberOfBytes() const675 vtkm::BufferSizeType Buffer::GetNumberOfBytes() const
676 {
677 LockType lock = this->Internals->GetLock();
678 return this->Internals->GetNumberOfBytes(lock);
679 }
680
SetNumberOfBytes(vtkm::BufferSizeType numberOfBytes,vtkm::CopyFlag preserve,vtkm::cont::Token & token)681 void Buffer::SetNumberOfBytes(vtkm::BufferSizeType numberOfBytes,
682 vtkm::CopyFlag preserve,
683 vtkm::cont::Token& token)
684 {
685 LockType lock = this->Internals->GetLock();
686 detail::BufferHelper::SetNumberOfBytes(this->Internals, lock, numberOfBytes, preserve, token);
687 }
688
HasMetaData() const689 bool Buffer::HasMetaData() const
690 {
691 return (this->Internals->MetaData.Data != nullptr);
692 }
693
MetaDataIsType(const std::string & type) const694 bool Buffer::MetaDataIsType(const std::string& type) const
695 {
696 return this->HasMetaData() && (this->Internals->MetaData.Type == type);
697 }
698
SetMetaData(void * data,const std::string & type,detail::DeleterType * deleter,detail::CopierType * copier) const699 void Buffer::SetMetaData(void* data,
700 const std::string& type,
701 detail::DeleterType* deleter,
702 detail::CopierType* copier) const
703 {
704 this->Internals->MetaData.Initialize(data, type, deleter, copier);
705 }
706
GetMetaData(const std::string & type) const707 void* Buffer::GetMetaData(const std::string& type) const
708 {
709 if (type != this->Internals->MetaData.Type)
710 {
711 throw vtkm::cont::ErrorBadType("Requesting Buffer meta data that is the wrong type.");
712 }
713 return this->Internals->MetaData.Data;
714 }
715
IsAllocatedOnHost() const716 bool Buffer::IsAllocatedOnHost() const
717 {
718 LockType lock = this->Internals->GetLock();
719 return this->Internals->GetHostBuffer(lock).UpToDate;
720 }
721
IsAllocatedOnDevice(vtkm::cont::DeviceAdapterId device) const722 bool Buffer::IsAllocatedOnDevice(vtkm::cont::DeviceAdapterId device) const
723 {
724 if (device.IsValueValid())
725 {
726 LockType lock = this->Internals->GetLock();
727 return this->Internals->GetDeviceBuffers(lock)[device].UpToDate;
728 }
729 else if (device == vtkm::cont::DeviceAdapterTagUndefined{})
730 {
731 // Return if allocated on host.
732 return this->IsAllocatedOnHost();
733 }
734 else if (device == vtkm::cont::DeviceAdapterTagAny{})
735 {
736 // Return if allocated on any device.
737 LockType lock = this->Internals->GetLock();
738 for (auto&& deviceBuffer : this->Internals->GetDeviceBuffers(lock))
739 {
740 if (deviceBuffer.second.UpToDate)
741 {
742 return true;
743 }
744 }
745 return false;
746 }
747 else
748 {
749 // Should we throw an exception here?
750 return false;
751 }
752 }
753
ReadPointerHost(vtkm::cont::Token & token) const754 const void* Buffer::ReadPointerHost(vtkm::cont::Token& token) const
755 {
756 LockType lock = this->Internals->GetLock();
757 detail::BufferHelper::WaitToRead(this->Internals, lock, token);
758 detail::BufferHelper::AllocateOnHost(
759 this->Internals, lock, token, detail::BufferHelper::AccessMode::READ);
760 return this->Internals->GetHostBuffer(lock).GetPointer();
761 }
762
ReadPointerDevice(vtkm::cont::DeviceAdapterId device,vtkm::cont::Token & token) const763 const void* Buffer::ReadPointerDevice(vtkm::cont::DeviceAdapterId device,
764 vtkm::cont::Token& token) const
765 {
766 if (device.IsValueValid())
767 {
768 LockType lock = this->Internals->GetLock();
769 detail::BufferHelper::WaitToRead(this->Internals, lock, token);
770 detail::BufferHelper::AllocateOnDevice(
771 this->Internals, lock, token, device, detail::BufferHelper::AccessMode::READ);
772 return this->Internals->GetDeviceBuffers(lock)[device].GetPointer();
773 }
774 else if (device == vtkm::cont::DeviceAdapterTagUndefined{})
775 {
776 return this->ReadPointerHost(token);
777 }
778 else
779 {
780 throw vtkm::cont::ErrorBadDevice("Invalid device given to ReadPointerDevice");
781 }
782 }
783
WritePointerHost(vtkm::cont::Token & token) const784 void* Buffer::WritePointerHost(vtkm::cont::Token& token) const
785 {
786 LockType lock = this->Internals->GetLock();
787 detail::BufferHelper::WaitToWrite(this->Internals, lock, token);
788 detail::BufferHelper::AllocateOnHost(
789 this->Internals, lock, token, detail::BufferHelper::AccessMode::WRITE);
790
791 // Array is being written on host. All other buffers invalidated, so delete them.
792 for (auto&& deviceBuffer : this->Internals->GetDeviceBuffers(lock))
793 {
794 deviceBuffer.second.Release();
795 }
796
797 return this->Internals->GetHostBuffer(lock).GetPointer();
798 }
799
WritePointerDevice(vtkm::cont::DeviceAdapterId device,vtkm::cont::Token & token) const800 void* Buffer::WritePointerDevice(vtkm::cont::DeviceAdapterId device, vtkm::cont::Token& token) const
801 {
802 if (device.IsValueValid())
803 {
804 LockType lock = this->Internals->GetLock();
805 detail::BufferHelper::WaitToWrite(this->Internals, lock, token);
806 detail::BufferHelper::AllocateOnDevice(
807 this->Internals, lock, token, device, detail::BufferHelper::AccessMode::WRITE);
808
809 // Array is being written on this device. All other buffers invalided, so delete them.
810 this->Internals->GetHostBuffer(lock).Release();
811 for (auto&& deviceBuffer : this->Internals->GetDeviceBuffers(lock))
812 {
813 if (deviceBuffer.first != device)
814 {
815 deviceBuffer.second.Release();
816 }
817 }
818
819 return this->Internals->GetDeviceBuffers(lock)[device].GetPointer();
820 }
821 else if (device == vtkm::cont::DeviceAdapterTagUndefined{})
822 {
823 return this->WritePointerHost(token);
824 }
825 else
826 {
827 throw vtkm::cont::ErrorBadDevice("Invalid device given to WritePointerDevice");
828 }
829 }
830
Enqueue(const vtkm::cont::Token & token) const831 void Buffer::Enqueue(const vtkm::cont::Token& token) const
832 {
833 LockType lock = this->Internals->GetLock();
834 detail::BufferHelper::Enqueue(this->Internals, lock, token);
835 }
836
DeepCopyFrom(const vtkm::cont::internal::Buffer & src) const837 void Buffer::DeepCopyFrom(const vtkm::cont::internal::Buffer& src) const
838 {
839 // A Token should not be declared within the scope of a lock. when the token goes out of scope
840 // it will attempt to aquire the lock, which is undefined behavior of the thread already has
841 // the lock.
842 vtkm::cont::Token token;
843 {
844 const vtkm::cont::internal::Buffer& dest = *this;
845
846 LockType srcLock = src.Internals->GetLock();
847 LockType destLock = dest.Internals->GetLock();
848
849 detail::BufferHelper::WaitToRead(src.Internals, srcLock, token);
850
851 // If we are on a device, copy there.
852 for (auto&& deviceBuffer : src.Internals->GetDeviceBuffers(srcLock))
853 {
854 if (deviceBuffer.second.UpToDate)
855 {
856 detail::BufferHelper::CopyOnDevice(
857 deviceBuffer.first, src.Internals, srcLock, dest.Internals, destLock, token);
858 return;
859 }
860 }
861
862 // If we are here, there were no devices to copy on. Copy on host if possible.
863 if (src.Internals->GetHostBuffer(srcLock).UpToDate)
864 {
865 detail::BufferHelper::CopyOnHost(src.Internals, srcLock, dest.Internals, destLock, token);
866 }
867 else
868 {
869 // Nothing actually allocated. Just create allocation for dest. (Allocation happens lazily.)
870 detail::BufferHelper::SetNumberOfBytes(dest.Internals,
871 destLock,
872 src.Internals->GetNumberOfBytes(srcLock),
873 vtkm::CopyFlag::Off,
874 token);
875 dest.Internals->MetaData.DeepCopyFrom(src.Internals->MetaData);
876 }
877 }
878 }
879
DeepCopyFrom(const vtkm::cont::internal::Buffer & src,vtkm::cont::DeviceAdapterId device) const880 void Buffer::DeepCopyFrom(const vtkm::cont::internal::Buffer& src,
881 vtkm::cont::DeviceAdapterId device) const
882 {
883 // A Token should not be declared within the scope of a lock. when the token goes out of scope
884 // it will attempt to aquire the lock, which is undefined behavior of the thread already has
885 // the lock.
886 vtkm::cont::Token token;
887 {
888 LockType srcLock = src.Internals->GetLock();
889 LockType destLock = this->Internals->GetLock();
890 detail::BufferHelper::CopyOnDevice(
891 device, this->Internals, srcLock, this->Internals, destLock, token);
892 }
893 }
894
Reset(const vtkm::cont::internal::BufferInfo & bufferInfo)895 void Buffer::Reset(const vtkm::cont::internal::BufferInfo& bufferInfo)
896 {
897 LockType lock = this->Internals->GetLock();
898
899 // Clear out any old buffers. Because we are resetting the object, we will also get rid of
900 // pinned memory.
901 this->Internals->GetHostBuffer(lock) = BufferState{};
902 this->Internals->GetDeviceBuffers(lock).clear();
903
904 if (bufferInfo.GetDevice().IsValueValid())
905 {
906 this->Internals->GetDeviceBuffers(lock)[bufferInfo.GetDevice()] =
907 BufferState{ bufferInfo, true, true };
908 }
909 else if (bufferInfo.GetDevice() == vtkm::cont::DeviceAdapterTagUndefined{})
910 {
911 this->Internals->GetHostBuffer(lock) = BufferState{ bufferInfo, true, true };
912 }
913 else
914 {
915 this->Internals->SetNumberOfBytes(lock, 0);
916 throw vtkm::cont::ErrorBadDevice("Attempting to reset Buffer to invalid device.");
917 }
918
919 this->Internals->SetNumberOfBytes(lock, bufferInfo.GetSize());
920 }
921
ReleaseDeviceResources() const922 void Buffer::ReleaseDeviceResources() const
923 {
924 vtkm::cont::Token token;
925
926 // Getting a write host buffer will invalidate any device arrays and preserve data
927 // on the host (copying if necessary).
928 this->WritePointerHost(token);
929 }
930
GetHostBufferInfo() const931 vtkm::cont::internal::BufferInfo Buffer::GetHostBufferInfo() const
932 {
933 LockType lock = this->Internals->GetLock();
934 return this->Internals->GetHostBuffer(lock);
935 }
936
TakeHostBufferOwnership()937 vtkm::cont::internal::TransferredBuffer Buffer::TakeHostBufferOwnership()
938 {
939 // A Token should not be declared within the scope of a lock. when the token goes out of scope
940 // it will attempt to acquire the lock, which is undefined behavior of the thread already has
941 // the lock.
942 vtkm::cont::Token token;
943 {
944 LockType lock = this->Internals->GetLock();
945 detail::BufferHelper::AllocateOnHost(
946 this->Internals, lock, token, detail::BufferHelper::AccessMode::READ);
947 auto& buffer = this->Internals->GetHostBuffer(lock);
948 buffer.Pinned = true;
949 return buffer.Info.TransferOwnership();
950 }
951 }
952
TakeDeviceBufferOwnership(vtkm::cont::DeviceAdapterId device)953 vtkm::cont::internal::TransferredBuffer Buffer::TakeDeviceBufferOwnership(
954 vtkm::cont::DeviceAdapterId device)
955 {
956 if (device.IsValueValid())
957 {
958 // A Token should not be declared within the scope of a lock. when the token goes out of scope
959 // it will attempt to acquire the lock, which is undefined behavior of the thread already has
960 // the lock.
961 vtkm::cont::Token token;
962 {
963 LockType lock = this->Internals->GetLock();
964 detail::BufferHelper::AllocateOnDevice(
965 this->Internals, lock, token, device, detail::BufferHelper::AccessMode::READ);
966 auto& buffer = this->Internals->GetDeviceBuffers(lock)[device];
967 buffer.Pinned = true;
968 return buffer.Info.TransferOwnership();
969 }
970 }
971 else if (device == vtkm::cont::DeviceAdapterTagUndefined{})
972 {
973 return this->TakeHostBufferOwnership();
974 }
975 else
976 {
977 throw vtkm::cont::ErrorBadDevice(
978 "Called Buffer::TakeDeviceBufferOwnership with invalid device");
979 }
980 }
981
GetDeviceBufferInfo(vtkm::cont::DeviceAdapterId device) const982 vtkm::cont::internal::BufferInfo Buffer::GetDeviceBufferInfo(
983 vtkm::cont::DeviceAdapterId device) const
984 {
985 if (device.IsValueValid())
986 {
987 LockType lock = this->Internals->GetLock();
988 return this->Internals->GetDeviceBuffers(lock)[device];
989 }
990 else if (device == vtkm::cont::DeviceAdapterTagUndefined{})
991 {
992 return this->GetHostBufferInfo();
993 }
994 else
995 {
996 throw vtkm::cont::ErrorBadDevice("Called Buffer::GetDeviceBufferInfo with invalid device");
997 }
998 }
999 }
1000 }
1001 } // namespace vtkm::cont::internal
1002
1003 namespace mangled_diy_namespace
1004 {
1005
save(BinaryBuffer & bb,const vtkm::cont::internal::Buffer & obj)1006 void Serialization<vtkm::cont::internal::Buffer>::save(BinaryBuffer& bb,
1007 const vtkm::cont::internal::Buffer& obj)
1008 {
1009 vtkm::BufferSizeType size = obj.GetNumberOfBytes();
1010 vtkmdiy::save(bb, size);
1011
1012 vtkm::cont::Token token;
1013 const vtkm::UInt8* data = reinterpret_cast<const vtkm::UInt8*>(obj.ReadPointerHost(token));
1014 vtkmdiy::save(bb, data, static_cast<std::size_t>(size));
1015 }
1016
load(BinaryBuffer & bb,vtkm::cont::internal::Buffer & obj)1017 void Serialization<vtkm::cont::internal::Buffer>::load(BinaryBuffer& bb,
1018 vtkm::cont::internal::Buffer& obj)
1019 {
1020 vtkm::BufferSizeType size;
1021 vtkmdiy::load(bb, size);
1022
1023 vtkm::cont::Token token;
1024 obj.SetNumberOfBytes(size, vtkm::CopyFlag::Off, token);
1025 vtkm::UInt8* data = reinterpret_cast<vtkm::UInt8*>(obj.WritePointerHost(token));
1026 vtkmdiy::load(bb, data, static_cast<std::size_t>(size));
1027 }
1028
1029 } // namespace diy
1030