benchmarks/tbb/x86_rtm_rw_mutex.cpp

/*
    Copyright 2005-2014 Intel Corporation.  All Rights Reserved.

    This file is part of Threading Building Blocks. Threading Building Blocks is free software;
    you can redistribute it and/or modify it under the terms of the GNU General Public License
    version 2  as  published  by  the  Free Software Foundation.  Threading Building Blocks is
    distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
    implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
    See  the GNU General Public License for more details.   You should have received a copy of
    the  GNU General Public License along with Threading Building Blocks; if not, write to the
    Free Software Foundation, Inc.,  51 Franklin St,  Fifth Floor,  Boston,  MA 02110-1301 USA

    As a special exception,  you may use this file  as part of a free software library without
    restriction.  Specifically,  if other files instantiate templates  or use macros or inline
    functions from this file, or you compile this file and link it with other files to produce
    an executable,  this file does not by itself cause the resulting executable to be covered
    by the GNU General Public License. This exception does not however invalidate any other
    reasons why the executable file might be covered by the GNU General Public License.
*/

#include "tbb/tbb_config.h"
#if __TBB_TSX_AVAILABLE
#include "tbb/spin_rw_mutex.h"
#include "tbb/tbb_machine.h"
#include "itt_notify.h"
#include "governor.h"
#include "tbb/atomic.h"

// __TBB_RW_MUTEX_DELAY_TEST shifts the point where flags aborting speculation are
// added to the read-set of the operation.  If 1, will add the test just before
// the transaction is ended.
#ifndef __TBB_RW_MUTEX_DELAY_TEST
    #define __TBB_RW_MUTEX_DELAY_TEST 1
#endif

#if defined(_MSC_VER) && defined(_Wp64)
    // Workaround for overzealous compiler warnings in /Wp64 mode
    #pragma warning (disable: 4244)
#endif

namespace tbb {

namespace interface8 {
namespace internal {

// abort code for mutexes that detect a conflict with another thread.
// value is hexadecimal
enum {
    speculation_transaction_aborted = 0x01,
    speculation_can_retry           = 0x02,
    speculation_memadd_conflict     = 0x04,
    speculation_buffer_overflow     = 0x08,
    speculation_breakpoint_hit      = 0x10,
    speculation_nested_abort        = 0x20,
    speculation_xabort_mask         = 0xFF000000,
    speculation_xabort_shift        = 24,
    speculation_retry               = speculation_transaction_aborted
                                      | speculation_can_retry
                                      | speculation_memadd_conflict
};

// maximum number of times to retry
static const int retry_threshold_read = 10;
static const int retry_threshold_write = 10;

//! Release speculative mutex
void x86_rtm_rw_mutex::internal_release(x86_rtm_rw_mutex::scoped_lock& s) {
    switch(s.transaction_state) {
    case RTM_transacting_writer:
    case RTM_transacting_reader:
        {
            __TBB_ASSERT(__TBB_machine_is_in_transaction(), "transaction_state && not speculating");
#if __TBB_RW_MUTEX_DELAY_TEST
            if(s.transaction_state == RTM_transacting_reader) {
                if(this->w_flag) __TBB_machine_transaction_conflict_abort();
            } else {
                if(this->state) __TBB_machine_transaction_conflict_abort();
            }
#endif
            __TBB_machine_end_transaction();
            s.my_scoped_lock.internal_set_mutex(NULL);
        }
        break;
    case RTM_real_reader:
        __TBB_ASSERT(!this->w_flag, "w_flag set but read lock acquired");
        s.my_scoped_lock.release();
        break;
    case RTM_real_writer:
        __TBB_ASSERT(this->w_flag, "w_flag unset but write lock acquired");
        this->w_flag = false;
        s.my_scoped_lock.release();
        break;
    case RTM_not_in_mutex:
        __TBB_ASSERT(false, "RTM_not_in_mutex, but in release");
    default:
        __TBB_ASSERT(false, "invalid transaction_state");
    }
    s.transaction_state = RTM_not_in_mutex;
}

//! Acquire write lock on the given mutex.
void x86_rtm_rw_mutex::internal_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate)
{
    __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction");
    if(tbb::internal::governor::speculation_enabled()) {
        int num_retries = 0;
        unsigned int abort_code;
        do {
            tbb::internal::atomic_backoff backoff;
            if(this->state) {
                if(only_speculate) return;
                do {
                    backoff.pause();  // test the spin_rw_mutex (real readers or writers)
                } while(this->state);
            }
            // _xbegin returns -1 on success or the abort code, so capture it
            if(( abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) )
            {
                // started speculation
#if !__TBB_RW_MUTEX_DELAY_TEST
                if(this->state) {  // add spin_rw_mutex to read-set.
                    // reader or writer grabbed the lock, so abort.
                    __TBB_machine_transaction_conflict_abort();
                }
#endif
                s.transaction_state = RTM_transacting_writer;
                s.my_scoped_lock.internal_set_mutex(this);  // need mutex for release()
                return;  // successfully started speculation
            }
            ++num_retries;
        } while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_write) );
    }

    if(only_speculate) return;              // should apply a real try_lock...
    s.my_scoped_lock.acquire(*this, true);  // kill transactional writers
    __TBB_ASSERT(!w_flag, "After acquire for write, w_flag already true");
    w_flag = true;                          // kill transactional readers
    s.transaction_state = RTM_real_writer;
    return;
}

//! Acquire read lock on given mutex.
//  only_speculate : true if we are doing a try_acquire.  If true and we fail to speculate, don't
//     really acquire the lock, return and do a try_acquire on the contained spin_rw_mutex.  If
//     the lock is already held by a writer, just return.
void x86_rtm_rw_mutex::internal_acquire_reader(x86_rtm_rw_mutex::scoped_lock& s, bool only_speculate) {
    __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "scoped_lock already in transaction");
    if(tbb::internal::governor::speculation_enabled()) {
        int num_retries = 0;
        unsigned int abort_code;
        do {
            tbb::internal::atomic_backoff backoff;
            // if in try_acquire, and lock is held as writer, don't attempt to speculate.
            if(w_flag) {
                if(only_speculate) return;
                do {
                    backoff.pause();  // test the spin_rw_mutex (real readers or writers)
                } while(w_flag);
            }
            // _xbegin returns -1 on success or the abort code, so capture it
            if((abort_code = __TBB_machine_begin_transaction()) == ~(unsigned int)(0) )
            {
                // started speculation
#if !__TBB_RW_MUTEX_DELAY_TEST
                if(w_flag) {  // add w_flag to read-set.
                    __TBB_machine_transaction_conflict_abort();  // writer grabbed the lock, so abort.
                }
#endif
                s.transaction_state = RTM_transacting_reader;
                s.my_scoped_lock.internal_set_mutex(this);  // need mutex for release()
                return;  // successfully started speculation
            }
            // fallback path
            // retry only if there is any hope of getting into a transaction soon
            // Retry in the following cases (from Section 8.3.5 of Intel(R)
            // Architecture Instruction Set Extensions Programming Reference):
            // 1. abort caused by XABORT instruction (bit 0 of EAX register is set)
            // 2. the transaction may succeed on a retry (bit 1 of EAX register is set)
            // 3. if another logical processor conflicted with a memory address
            //    that was part of the transaction that aborted (bit 2 of EAX register is set)
            // That is, retry if (abort_code & 0x7) is non-zero
            ++num_retries;
        } while( (abort_code & speculation_retry) != 0 && (num_retries < retry_threshold_read) );
    }

    if(only_speculate) return;
    s.my_scoped_lock.acquire( *this, false );
    s.transaction_state = RTM_real_reader;
}

//! Upgrade reader to become a writer.
/** Returns whether the upgrade happened without releasing and re-acquiring the lock */
bool x86_rtm_rw_mutex::internal_upgrade(x86_rtm_rw_mutex::scoped_lock& s)
{
    switch(s.transaction_state) {
    case RTM_real_reader: {
            s.transaction_state = RTM_real_writer;
            bool no_release = s.my_scoped_lock.upgrade_to_writer();
            __TBB_ASSERT(!w_flag, "After upgrade_to_writer, w_flag already true");
            w_flag = true;
            return no_release;
        }
    case RTM_transacting_reader:
        s.transaction_state = RTM_transacting_writer;
        // don't need to add w_flag to read_set even if __TBB_RW_MUTEX_DELAY_TEST
        // because the this pointer (the spin_rw_mutex) will be sufficient on release.
        return true;
    default:
        __TBB_ASSERT(false, "Invalid state for upgrade");
        return false;
    }
}

//! Downgrade writer to a reader.
bool x86_rtm_rw_mutex::internal_downgrade(x86_rtm_rw_mutex::scoped_lock& s) {
    switch(s.transaction_state) {
    case RTM_real_writer:
        s.transaction_state = RTM_real_reader;
        __TBB_ASSERT(w_flag, "Before downgrade_to_reader w_flag not true");
        w_flag = false;
        return s.my_scoped_lock.downgrade_to_reader();
    case RTM_transacting_writer:
#if __TBB_RW_MUTEX_DELAY_TEST
        if(this->state) {  // a reader or writer has acquired mutex for real.
            __TBB_machine_transaction_conflict_abort();
        }
#endif
        s.transaction_state = RTM_transacting_reader;
        return true;
    default:
        __TBB_ASSERT(false, "Invalid state for downgrade");
        return false;
    }
}

//! Try to acquire write lock on the given mutex.
//  There may be reader(s) which acquired the spin_rw_mutex, as well as possibly
//  transactional reader(s).  If this is the case, the acquire will fail, and assigning
//  w_flag will kill the transactors.  So we only assign w_flag if we have successfully
//  acquired the lock.
bool x86_rtm_rw_mutex::internal_try_acquire_writer(x86_rtm_rw_mutex::scoped_lock& s)
{
    internal_acquire_writer(s, /*only_speculate=*/true);
    if(s.transaction_state == RTM_transacting_writer) {
        return true;
    }
    __TBB_ASSERT(s.transaction_state == RTM_not_in_mutex, "Trying to acquire writer which is already allocated");
    // transacting write acquire failed.  try_acquire the real mutex
    bool result = s.my_scoped_lock.try_acquire(*this, true);
    if(result) {
        // only shoot down readers if we're not transacting ourselves
        __TBB_ASSERT(!w_flag, "After try_acquire_writer, w_flag already true");
        w_flag = true;
        s.transaction_state = RTM_real_writer;
    }
    return result;
}

void x86_rtm_rw_mutex::internal_construct() {
    ITT_SYNC_CREATE(this, _T("tbb::x86_rtm_rw_mutex"), _T(""));
}

} // namespace internal
} // namespace interface8
} // namespace tbb

#endif /* __TBB_TSX_AVAILABLE */