src/jit/AtomicOperations.h

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
 * vim: set ts=8 sts=4 et sw=4 tw=99:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_AtomicOperations_h
#define jit_AtomicOperations_h

#include "mozilla/Types.h"

#include "vm/SharedMem.h"

namespace js {
namespace jit {

class RegionLock;

/*
 * The atomic operations layer defines types and functions for
 * JIT-compatible atomic operation.
 *
 * The fundamental constraints on the functions are:
 *
 * - That their realization here MUST be compatible with code the JIT
 *   generates for its Atomics operations, so that an atomic access
 *   from the interpreter or runtime - from any C++ code - really is
 *   atomic relative to a concurrent, compatible atomic access from
 *   jitted code.  That is, these primitives expose JIT-compatible
 *   atomicity functionality to C++.
 *
 * - That accesses may race without creating C++ undefined behavior:
 *   atomic accesses (marked "SeqCst") may race with non-atomic
 *   accesses (marked "SafeWhenRacy"); overlapping but non-matching,
 *   and hence incompatible, atomic accesses may race; and non-atomic
 *   accesses may race.  The effects of races need not be predictable,
 *   so garbage can be produced by a read or written by a write, but
 *   the effects must be benign: the program must continue to run, and
 *   only the memory in the union of addresses named in the racing
 *   accesses may be affected.
 *
 * The compatibility constraint means that if the JIT makes dynamic
 * decisions about how to implement atomic operations then
 * corresponding dynamic decisions MUST be made in the implementations
 * of the functions below.
 *
 * The safe-for-races constraint means that by and large, it is hard
 * to implement these primitives in C++.  See "Implementation notes"
 * below.
 *
 * The "SeqCst" suffix on operations means "sequentially consistent"
 * and means such a function's operation must have "sequentially
 * consistent" memory ordering.  See mfbt/Atomics.h for an explanation
 * of this memory ordering.
 *
 * Note that a "SafeWhenRacy" access does not provide the atomicity of
 * a "relaxed atomic" access: it can read or write garbage if there's
 * a race.
 *
 *
 * Implementation notes.
 *
 * It's not a requirement that these functions be inlined; performance
 * is not a great concern.  On some platforms these functions may call
 * out to code that's generated at run time.
 *
 * In principle these functions will not be written in C++, thus
 * making races defined behavior if all racy accesses from C++ go via
 * these functions.  (Jitted code will always be safe for races and
 * provides the same guarantees as these functions.)
 *
 * The appropriate implementations will be platform-specific and
 * there are some obvious implementation strategies to choose
 * from, sometimes a combination is appropriate:
 *
 *  - generating the code at run-time with the JIT;
 *  - hand-written assembler (maybe inline); or
 *  - using special compiler intrinsics or directives.
 *
 * Trusting the compiler not to generate code that blows up on a
 * race definitely won't work in the presence of TSan, or even of
 * optimizing compilers in seemingly-"innocuous" conditions.  (See
 * https://www.usenix.org/legacy/event/hotpar11/tech/final_files/Boehm.pdf
 * for details.)
 */
class AtomicOperations
{
    friend class RegionLock;

  private:
    // The following functions are defined for T = int8_t, uint8_t,
    // int16_t, uint16_t, int32_t, uint32_t, int64_t, and uint64_t.

    // Atomically read *addr.
    template<typename T>
    static inline T loadSeqCst(T* addr);

    // Atomically store val in *addr.
    template<typename T>
    static inline void storeSeqCst(T* addr, T val);

    // Atomically store val in *addr and return the old value of *addr.
    template<typename T>
    static inline T exchangeSeqCst(T* addr, T val);

    // Atomically check that *addr contains oldval and if so replace it
    // with newval, in any case returning the old contents of *addr.
    template<typename T>
    static inline T compareExchangeSeqCst(T* addr, T oldval, T newval);

    // The following functions are defined for T = int8_t, uint8_t,
    // int16_t, uint16_t, int32_t, uint32_t only.

    // Atomically add, subtract, bitwise-AND, bitwise-OR, or bitwise-XOR
    // val into *addr and return the old value of *addr.
    template<typename T>
    static inline T fetchAddSeqCst(T* addr, T val);

    template<typename T>
    static inline T fetchSubSeqCst(T* addr, T val);

    template<typename T>
    static inline T fetchAndSeqCst(T* addr, T val);

    template<typename T>
    static inline T fetchOrSeqCst(T* addr, T val);

    template<typename T>
    static inline T fetchXorSeqCst(T* addr, T val);

    // The SafeWhenRacy functions are to be used when C++ code has to access
    // memory without synchronization and can't guarantee that there
    // won't be a race on the access.

    // Defined for all the integral types as well as for float32 and float64.
    template<typename T>
    static inline T loadSafeWhenRacy(T* addr);

    // Defined for all the integral types as well as for float32 and float64.
    template<typename T>
    static inline void storeSafeWhenRacy(T* addr, T val);

    // Replacement for memcpy().
    static inline void memcpySafeWhenRacy(void* dest, const void* src, size_t nbytes);

    // Replacement for memmove().
    static inline void memmoveSafeWhenRacy(void* dest, const void* src, size_t nbytes);

  public:
    // Test lock-freedom for any integer value.
    //
    // This implements a platform-independent pattern, as follows:
    //
    // 1, 2, and 4 bytes are always lock free, lock-freedom for 8
    // bytes is determined by the platform's isLockfree8(), and there
    // is no lock-freedom for any other values on any platform.
    static inline bool isLockfree(int32_t n);

    // If the return value is true then a call to the 64-bit (8-byte)
    // routines below will work, otherwise those functions will assert in
    // debug builds and may crash in release build.  (See the code in
    // ../arm for an example.)  The value of this call does not change
    // during execution.
    static inline bool isLockfree8();

    // Execute a full memory barrier (LoadLoad+LoadStore+StoreLoad+StoreStore).
    static inline void fenceSeqCst();

    // All clients should use the APIs that take SharedMem pointers.
    // See above for semantics and acceptable types.

    template<typename T>
    static T loadSeqCst(SharedMem<T*> addr) {
        return loadSeqCst(addr.unwrap());
    }

    template<typename T>
    static void storeSeqCst(SharedMem<T*> addr, T val) {
        return storeSeqCst(addr.unwrap(), val);
    }

    template<typename T>
    static T exchangeSeqCst(SharedMem<T*> addr, T val) {
        return exchangeSeqCst(addr.unwrap(), val);
    }

    template<typename T>
    static T compareExchangeSeqCst(SharedMem<T*> addr, T oldval, T newval) {
        return compareExchangeSeqCst(addr.unwrap(), oldval, newval);
    }

    template<typename T>
    static T fetchAddSeqCst(SharedMem<T*> addr, T val) {
        return fetchAddSeqCst(addr.unwrap(), val);
    }

    template<typename T>
    static T fetchSubSeqCst(SharedMem<T*> addr, T val) {
        return fetchSubSeqCst(addr.unwrap(), val);
    }

    template<typename T>
    static T fetchAndSeqCst(SharedMem<T*> addr, T val) {
        return fetchAndSeqCst(addr.unwrap(), val);
    }

    template<typename T>
    static T fetchOrSeqCst(SharedMem<T*> addr, T val) {
        return fetchOrSeqCst(addr.unwrap(), val);
    }

    template<typename T>
    static T fetchXorSeqCst(SharedMem<T*> addr, T val) {
        return fetchXorSeqCst(addr.unwrap(), val);
    }

    template<typename T>
    static T loadSafeWhenRacy(SharedMem<T*> addr) {
        return loadSafeWhenRacy(addr.unwrap());
    }

    template<typename T>
    static void storeSafeWhenRacy(SharedMem<T*> addr, T val) {
        return storeSafeWhenRacy(addr.unwrap(), val);
    }

    template<typename T>
    static void memcpySafeWhenRacy(SharedMem<T> dest, SharedMem<T> src, size_t nbytes) {
        memcpySafeWhenRacy(static_cast<void*>(dest.unwrap()), static_cast<void*>(src.unwrap()), nbytes);
    }

    template<typename T>
    static void memcpySafeWhenRacy(SharedMem<T> dest, T src, size_t nbytes) {
        memcpySafeWhenRacy(static_cast<void*>(dest.unwrap()), static_cast<void*>(src), nbytes);
    }

    template<typename T>
    static void memcpySafeWhenRacy(T dest, SharedMem<T> src, size_t nbytes) {
        memcpySafeWhenRacy(static_cast<void*>(dest), static_cast<void*>(src.unwrap()), nbytes);
    }

    template<typename T>
    static void memmoveSafeWhenRacy(SharedMem<T> dest, SharedMem<T> src, size_t nbytes) {
        memmoveSafeWhenRacy(static_cast<void*>(dest.unwrap()), static_cast<void*>(src.unwrap()), nbytes);
    }
};

/* A data type representing a lock on some region of a
 * SharedArrayRawBuffer's memory, to be used only when the hardware
 * does not provide necessary atomicity (eg, float64 access on ARMv6
 * and some ARMv7 systems).
 */
class RegionLock
{
  public:
    RegionLock() : spinlock(0) {}

    /* Addr is the address to be locked, nbytes the number of bytes we
     * need to lock.  The lock that is taken may cover a larger range
     * of bytes.
     */
    template<size_t nbytes>
    void acquire(void* addr);

    /* Addr is the address to be unlocked, nbytes the number of bytes
     * we need to unlock.  The lock must be held by the calling thread,
     * at the given address and for the number of bytes.
     */
    template<size_t nbytes>
    void release(void* addr);

  private:
    /* For now, a simple spinlock that covers the entire buffer. */
    uint32_t spinlock;
};

inline bool
AtomicOperations::isLockfree(int32_t size)
{
    // Keep this in sync with visitAtomicIsLockFree() in jit/CodeGenerator.cpp.

    switch (size) {
      case 1:
      case 2:
      case 4:
        return true;
      case 8:
        return AtomicOperations::isLockfree8();
      default:
        return false;
    }
}

} // namespace jit
} // namespace js

#if defined(JS_CODEGEN_ARM)
# include "jit/arm/AtomicOperations-arm.h"
#elif defined(JS_CODEGEN_ARM64)
# include "jit/arm64/AtomicOperations-arm64.h"
#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
# include "jit/mips-shared/AtomicOperations-mips-shared.h"
#elif defined(__ppc64__) || defined(__PPC64_)       \
    || defined(__ppc64le__) || defined(__PPC64LE__) \
    || defined(__ppc__) || defined(__PPC__)
# include "jit/none/AtomicOperations-ppc.h"
#elif defined(JS_CODEGEN_NONE)
# include "jit/none/AtomicOperations-none.h"
#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
# include "jit/x86-shared/AtomicOperations-x86-shared.h"
#else
# error "Atomic operations must be defined for this platform"
#endif

#endif // jit_AtomicOperations_h