1 //===-- tsan_vector_clock.cpp ---------------------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 //===----------------------------------------------------------------------===// 12 #include "tsan_vector_clock.h" 13 14 #include "sanitizer_common/sanitizer_placement_new.h" 15 #include "tsan_mman.h" 16 17 namespace __tsan { 18 19 #if TSAN_VECTORIZE 20 const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128); 21 #endif 22 23 VectorClock::VectorClock() { Reset(); } 24 25 void VectorClock::Reset() { 26 #if !TSAN_VECTORIZE 27 for (uptr i = 0; i < kThreadSlotCount; i++) 28 clk_[i] = kEpochZero; 29 #else 30 m128 z = _mm_setzero_si128(); 31 m128* vclk = reinterpret_cast<m128*>(clk_); 32 for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z); 33 #endif 34 } 35 36 void VectorClock::Acquire(const VectorClock* src) { 37 if (!src) 38 return; 39 #if !TSAN_VECTORIZE 40 for (uptr i = 0; i < kThreadSlotCount; i++) 41 clk_[i] = max(clk_[i], src->clk_[i]); 42 #else 43 m128* __restrict vdst = reinterpret_cast<m128*>(clk_); 44 m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_); 45 for (uptr i = 0; i < kVectorClockSize; i++) { 46 m128 s = _mm_load_si128(&vsrc[i]); 47 m128 d = _mm_load_si128(&vdst[i]); 48 m128 m = _mm_max_epu16(s, d); 49 _mm_store_si128(&vdst[i], m); 50 } 51 #endif 52 } 53 54 static VectorClock* AllocClock(VectorClock** dstp) { 55 if (UNLIKELY(!*dstp)) 56 *dstp = New<VectorClock>(); 57 return *dstp; 58 } 59 60 void VectorClock::Release(VectorClock** dstp) const { 61 VectorClock* dst = AllocClock(dstp); 62 dst->Acquire(this); 63 } 64 65 void VectorClock::ReleaseStore(VectorClock** dstp) const { 66 VectorClock* dst = AllocClock(dstp); 67 *dst = *this; 68 } 69 70 VectorClock& VectorClock::operator=(const VectorClock& other) { 71 #if !TSAN_VECTORIZE 72 for (uptr i = 0; i < kThreadSlotCount; i++) 73 clk_[i] = other.clk_[i]; 74 #else 75 m128* __restrict vdst = reinterpret_cast<m128*>(clk_); 76 m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_); 77 for (uptr i = 0; i < kVectorClockSize; i++) { 78 m128 s = _mm_load_si128(&vsrc[i]); 79 _mm_store_si128(&vdst[i], s); 80 } 81 #endif 82 return *this; 83 } 84 85 void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) { 86 VectorClock* dst = AllocClock(dstp); 87 #if !TSAN_VECTORIZE 88 for (uptr i = 0; i < kThreadSlotCount; i++) { 89 Epoch tmp = dst->clk_[i]; 90 dst->clk_[i] = clk_[i]; 91 clk_[i] = max(clk_[i], tmp); 92 } 93 #else 94 m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_); 95 m128* __restrict vclk = reinterpret_cast<m128*>(clk_); 96 for (uptr i = 0; i < kVectorClockSize; i++) { 97 m128 t = _mm_load_si128(&vdst[i]); 98 m128 c = _mm_load_si128(&vclk[i]); 99 m128 m = _mm_max_epu16(c, t); 100 _mm_store_si128(&vdst[i], c); 101 _mm_store_si128(&vclk[i], m); 102 } 103 #endif 104 } 105 106 void VectorClock::ReleaseAcquire(VectorClock** dstp) { 107 VectorClock* dst = AllocClock(dstp); 108 #if !TSAN_VECTORIZE 109 for (uptr i = 0; i < kThreadSlotCount; i++) { 110 dst->clk_[i] = max(dst->clk_[i], clk_[i]); 111 clk_[i] = dst->clk_[i]; 112 } 113 #else 114 m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_); 115 m128* __restrict vclk = reinterpret_cast<m128*>(clk_); 116 for (uptr i = 0; i < kVectorClockSize; i++) { 117 m128 c = _mm_load_si128(&vclk[i]); 118 m128 d = _mm_load_si128(&vdst[i]); 119 m128 m = _mm_max_epu16(c, d); 120 _mm_store_si128(&vdst[i], m); 121 _mm_store_si128(&vclk[i], m); 122 } 123 #endif 124 } 125 126 } // namespace __tsan 127