1 /*
2     Copyright (c) 2005-2021 Intel Corporation
3 
4     Licensed under the Apache License, Version 2.0 (the "License");
5     you may not use this file except in compliance with the License.
6     You may obtain a copy of the License at
7 
8         http://www.apache.org/licenses/LICENSE-2.0
9 
10     Unless required by applicable law or agreed to in writing, software
11     distributed under the License is distributed on an "AS IS" BASIS,
12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13     See the License for the specific language governing permissions and
14     limitations under the License.
15 */
16 
17 #include "oneapi/tbb/detail/_assert.h"
18 #include "oneapi/tbb/detail/_rtm_mutex.h"
19 #include "itt_notify.h"
20 #include "governor.h"
21 #include "misc.h"
22 
23 #include <atomic>
24 
25 namespace tbb {
26 namespace detail {
27 namespace r1 {
28 
29 
30 struct rtm_mutex_impl {
31     // maximum number of times to retry
32     // TODO: experiment on retry values.
33     static constexpr int retry_threshold = 10;
34     using transaction_result_type = decltype(begin_transaction());
35 
36     //! Release speculative mutex
releasetbb::detail::r1::rtm_mutex_impl37     static void release(d1::rtm_mutex::scoped_lock& s) {
38         switch(s.m_transaction_state) {
39         case d1::rtm_mutex::rtm_state::rtm_transacting:
40             __TBB_ASSERT(is_in_transaction(), "m_transaction_state && not speculating");
41             end_transaction();
42             s.m_mutex = nullptr;
43             break;
44         case d1::rtm_mutex::rtm_state::rtm_real:
45             s.m_mutex->unlock();
46             s.m_mutex = nullptr;
47             break;
48         case d1::rtm_mutex::rtm_state::rtm_none:
49             __TBB_ASSERT(false, "mutex is not locked, but in release");
50             break;
51         default:
52             __TBB_ASSERT(false, "invalid m_transaction_state");
53         }
54         s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_none;
55     }
56 
57     //! Acquire lock on the given mutex.
acquiretbb::detail::r1::rtm_mutex_impl58     static void acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) {
59         __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, "scoped_lock already in transaction");
60         if(governor::speculation_enabled()) {
61             int num_retries = 0;
62             transaction_result_type abort_code = 0;
63             do {
64                 if(m.m_flag.load(std::memory_order_acquire)) {
65                     if(only_speculate) return;
66                     spin_wait_while_eq(m.m_flag, true);
67                 }
68                 // _xbegin returns -1 on success or the abort code, so capture it
69                 if((abort_code = begin_transaction()) == transaction_result_type(speculation_successful_begin))
70                 {
71                     // started speculation
72                     if(m.m_flag.load(std::memory_order_relaxed)) {
73                         abort_transaction();
74                     }
75                     s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_transacting;
76                     // Don not wrap the following assignment to a function,
77                     // because it can abort the transaction in debug. Need mutex for release().
78                     s.m_mutex = &m;
79                     return;  // successfully started speculation
80                 }
81                 ++num_retries;
82             } while((abort_code & speculation_retry) != 0 && (num_retries < retry_threshold));
83         }
84 
85         if(only_speculate) return;
86         s.m_mutex = &m;
87         s.m_mutex->lock();
88         s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real;
89         return;
90     }
91 
92     //! Try to acquire lock on the given mutex.
try_acquiretbb::detail::r1::rtm_mutex_impl93     static bool try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) {
94         acquire(m, s, /*only_speculate=*/true);
95         if (s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_transacting) {
96             return true;
97         }
98         __TBB_ASSERT(s.m_transaction_state == d1::rtm_mutex::rtm_state::rtm_none, NULL);
99         // transacting acquire failed. try_lock the real mutex
100         if (m.try_lock()) {
101             s.m_mutex = &m;
102             s.m_transaction_state = d1::rtm_mutex::rtm_state::rtm_real;
103             return true;
104         }
105         return false;
106     }
107 };
108 
acquire(d1::rtm_mutex & m,d1::rtm_mutex::scoped_lock & s,bool only_speculate)109 void __TBB_EXPORTED_FUNC acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s, bool only_speculate) {
110     rtm_mutex_impl::acquire(m, s, only_speculate);
111 }
try_acquire(d1::rtm_mutex & m,d1::rtm_mutex::scoped_lock & s)112 bool __TBB_EXPORTED_FUNC try_acquire(d1::rtm_mutex& m, d1::rtm_mutex::scoped_lock& s) {
113     return rtm_mutex_impl::try_acquire(m, s);
114 }
release(d1::rtm_mutex::scoped_lock & s)115 void __TBB_EXPORTED_FUNC release(d1::rtm_mutex::scoped_lock& s) {
116     rtm_mutex_impl::release(s);
117 }
118 
119 } // namespace r1
120 } // namespace detail
121 } // namespace tbb
122 
123