libdruntime/core/atomic.d

/**
 * The atomic module provides basic support for lock-free
 * concurrent programming.
 *
 * Copyright: Copyright Sean Kelly 2005 - 2016.
 * License:   $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
 * Authors:   Sean Kelly, Alex Rønne Petersen, Manu Evans
 * Source:    $(DRUNTIMESRC core/_atomic.d)
 */

module core.atomic;

import core.internal.atomic;
import core.internal.attributes : betterC;
import core.internal.traits : hasUnsharedIndirections;

/**
 * Specifies the memory ordering semantics of an atomic operation.
 *
 * See_Also:
 *     $(HTTP en.cppreference.com/w/cpp/atomic/memory_order)
 */
enum MemoryOrder
{
    /**
     * Not sequenced.
     * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#monotonic, LLVM AtomicOrdering.Monotonic)
     * and C++11/C11 `memory_order_relaxed`.
     */
    raw = 0,
    /**
     * Hoist-load + hoist-store barrier.
     * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#acquire, LLVM AtomicOrdering.Acquire)
     * and C++11/C11 `memory_order_acquire`.
     */
    acq = 2,
    /**
     * Sink-load + sink-store barrier.
     * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#release, LLVM AtomicOrdering.Release)
     * and C++11/C11 `memory_order_release`.
     */
    rel = 3,
    /**
     * Acquire + release barrier.
     * Corresponds to $(LINK2 https://llvm.org/docs/Atomics.html#acquirerelease, LLVM AtomicOrdering.AcquireRelease)
     * and C++11/C11 `memory_order_acq_rel`.
     */
    acq_rel = 4,
    /**
     * Fully sequenced (acquire + release). Corresponds to
     * $(LINK2 https://llvm.org/docs/Atomics.html#sequentiallyconsistent, LLVM AtomicOrdering.SequentiallyConsistent)
     * and C++11/C11 `memory_order_seq_cst`.
     */
    seq = 5,
}

/**
 * Loads 'val' from memory and returns it.  The memory barrier specified
 * by 'ms' is applied to the operation, which is fully sequenced by
 * default.  Valid memory orders are MemoryOrder.raw, MemoryOrder.acq,
 * and MemoryOrder.seq.
 *
 * Params:
 *  val = The target variable.
 *
 * Returns:
 *  The value of 'val'.
 */
T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)(ref return scope const T val) pure nothrow @nogc @trusted
    if (!is(T == shared U, U) && !is(T == shared inout U, U) && !is(T == shared const U, U))
{
    static if (__traits(isFloating, T))
    {
        alias IntTy = IntForFloat!T;
        IntTy r = core.internal.atomic.atomicLoad!ms(cast(IntTy*)&val);
        return *cast(T*)&r;
    }
    else
        return core.internal.atomic.atomicLoad!ms(cast(T*)&val);
}

/// Ditto
T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)(ref return scope shared const T val) pure nothrow @nogc @trusted
    if (!hasUnsharedIndirections!T)
{
    import core.internal.traits : hasUnsharedIndirections;
    static assert(!hasUnsharedIndirections!T, "Copying `" ~ shared(const(T)).stringof ~ "` would violate shared.");

    return atomicLoad!ms(*cast(T*)&val);
}

/// Ditto
TailShared!T atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)(ref shared const T val) pure nothrow @nogc @trusted
    if (hasUnsharedIndirections!T)
{
    // HACK: DEPRECATE THIS FUNCTION, IT IS INVALID TO DO ATOMIC LOAD OF SHARED CLASS
    // this is here because code exists in the wild that does this...

    return core.internal.atomic.atomicLoad!ms(cast(TailShared!T*)&val);
}

/**
 * Writes 'newval' into 'val'.  The memory barrier specified by 'ms' is
 * applied to the operation, which is fully sequenced by default.
 * Valid memory orders are MemoryOrder.raw, MemoryOrder.rel, and
 * MemoryOrder.seq.
 *
 * Params:
 *  val    = The target variable.
 *  newval = The value to store.
 */
void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V)(ref T val, V newval) pure nothrow @nogc @trusted
    if (!is(T == shared) && !is(V == shared))
{
    import core.internal.traits : hasElaborateCopyConstructor;
    static assert (!hasElaborateCopyConstructor!T, "`T` may not have an elaborate copy: atomic operations override regular copying semantics.");

    // resolve implicit conversions
    T arg = newval;

    static if (__traits(isFloating, T))
    {
        alias IntTy = IntForFloat!T;
        core.internal.atomic.atomicStore!ms(cast(IntTy*)&val, *cast(IntTy*)&arg);
    }
    else
        core.internal.atomic.atomicStore!ms(&val, arg);
}

/// Ditto
void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V)(ref shared T val, V newval) pure nothrow @nogc @trusted
    if (!is(T == class))
{
    static if (is (V == shared U, U))
        alias Thunk = U;
    else
    {
        import core.internal.traits : hasUnsharedIndirections;
        static assert(!hasUnsharedIndirections!V, "Copying argument `" ~ V.stringof ~ " newval` to `" ~ shared(T).stringof ~ " here` would violate shared.");
        alias Thunk = V;
    }
    atomicStore!ms(*cast(T*)&val, *cast(Thunk*)&newval);
}

/// Ditto
void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V)(ref shared T val, shared V newval) pure nothrow @nogc @trusted
    if (is(T == class))
{
    static assert (is (V : T), "Can't assign `newval` of type `shared " ~ V.stringof ~ "` to `shared " ~ T.stringof ~ "`.");

    core.internal.atomic.atomicStore!ms(cast(T*)&val, cast(V)newval);
}

/**
 * Atomically adds `mod` to the value referenced by `val` and returns the value `val` held previously.
 * This operation is both lock-free and atomic.
 *
 * Params:
 *  val = Reference to the value to modify.
 *  mod = The value to add.
 *
 * Returns:
 *  The value held previously by `val`.
 */
T atomicFetchAdd(MemoryOrder ms = MemoryOrder.seq, T)(ref return scope T val, size_t mod) pure nothrow @nogc @trusted
    if ((__traits(isIntegral, T) || is(T == U*, U)) && !is(T == shared))
in (atomicValueIsProperlyAligned(val))
{
    static if (is(T == U*, U))
        return cast(T)core.internal.atomic.atomicFetchAdd!ms(cast(size_t*)&val, mod * U.sizeof);
    else
        return core.internal.atomic.atomicFetchAdd!ms(&val, cast(T)mod);
}

/// Ditto
T atomicFetchAdd(MemoryOrder ms = MemoryOrder.seq, T)(ref return scope shared T val, size_t mod) pure nothrow @nogc @trusted
    if (__traits(isIntegral, T) || is(T == U*, U))
in (atomicValueIsProperlyAligned(val))
{
    return atomicFetchAdd!ms(*cast(T*)&val, mod);
}

/**
 * Atomically subtracts `mod` from the value referenced by `val` and returns the value `val` held previously.
 * This operation is both lock-free and atomic.
 *
 * Params:
 *  val = Reference to the value to modify.
 *  mod = The value to subtract.
 *
 * Returns:
 *  The value held previously by `val`.
 */
T atomicFetchSub(MemoryOrder ms = MemoryOrder.seq, T)(ref return scope T val, size_t mod) pure nothrow @nogc @trusted
    if ((__traits(isIntegral, T) || is(T == U*, U)) && !is(T == shared))
in (atomicValueIsProperlyAligned(val))
{
    static if (is(T == U*, U))
        return cast(T)core.internal.atomic.atomicFetchSub!ms(cast(size_t*)&val, mod * U.sizeof);
    else
        return core.internal.atomic.atomicFetchSub!ms(&val, cast(T)mod);
}

/// Ditto
T atomicFetchSub(MemoryOrder ms = MemoryOrder.seq, T)(ref return scope shared T val, size_t mod) pure nothrow @nogc @trusted
    if (__traits(isIntegral, T) || is(T == U*, U))
in (atomicValueIsProperlyAligned(val))
{
    return atomicFetchSub!ms(*cast(T*)&val, mod);
}

/**
 * Exchange `exchangeWith` with the memory referenced by `here`.
 * This operation is both lock-free and atomic.
 *
 * Params:
 *  here         = The address of the destination variable.
 *  exchangeWith = The value to exchange.
 *
 * Returns:
 *  The value held previously by `here`.
 */
T atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)(T* here, V exchangeWith) pure nothrow @nogc @trusted
    if (!is(T == shared) && !is(V == shared))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    // resolve implicit conversions
    T arg = exchangeWith;

    static if (__traits(isFloating, T))
    {
        alias IntTy = IntForFloat!T;
        IntTy r = core.internal.atomic.atomicExchange!ms(cast(IntTy*)here, *cast(IntTy*)&arg);
        return *cast(shared(T)*)&r;
    }
    else
        return core.internal.atomic.atomicExchange!ms(here, arg);
}

/// Ditto
TailShared!T atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)(shared(T)* here, V exchangeWith) pure nothrow @nogc @trusted
    if (!is(T == class) && !is(T == interface))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    static if (is (V == shared U, U))
        alias Thunk = U;
    else
    {
        import core.internal.traits : hasUnsharedIndirections;
        static assert(!hasUnsharedIndirections!V, "Copying `exchangeWith` of type `" ~ V.stringof ~ "` to `" ~ shared(T).stringof ~ "` would violate shared.");
        alias Thunk = V;
    }
    return atomicExchange!ms(cast(T*)here, *cast(Thunk*)&exchangeWith);
}

/// Ditto
shared(T) atomicExchange(MemoryOrder ms = MemoryOrder.seq,T,V)(shared(T)* here, shared(V) exchangeWith) pure nothrow @nogc @trusted
    if (is(T == class) || is(T == interface))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    static assert (is (V : T), "Can't assign `exchangeWith` of type `" ~ shared(V).stringof ~ "` to `" ~ shared(T).stringof ~ "`.");

    return cast(shared)core.internal.atomic.atomicExchange!ms(cast(T*)here, cast(V)exchangeWith);
}

/**
 * Performs either compare-and-set or compare-and-swap (or exchange).
 *
 * There are two categories of overloads in this template:
 * The first category does a simple compare-and-set.
 * The comparison value (`ifThis`) is treated as an rvalue.
 *
 * The second category does a compare-and-swap (a.k.a. compare-and-exchange),
 * and expects `ifThis` to be a pointer type, where the previous value
 * of `here` will be written.
 *
 * This operation is both lock-free and atomic.
 *
 * Params:
 *  here      = The address of the destination variable.
 *  writeThis = The value to store.
 *  ifThis    = The comparison value.
 *
 * Returns:
 *  true if the store occurred, false if not.
 */
template cas(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq)
{
    /// Compare-and-set for non-shared values
    bool cas(T, V1, V2)(T* here, V1 ifThis, V2 writeThis) pure nothrow @nogc @trusted
    if (!is(T == shared) && is(T : V1))
    in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
    {
        // resolve implicit conversions
        T arg1 = ifThis;
        T arg2 = writeThis;

        static if (__traits(isFloating, T))
        {
            alias IntTy = IntForFloat!T;
            return atomicCompareExchangeStrongNoResult!(succ, fail)(
                cast(IntTy*)here, *cast(IntTy*)&arg1, *cast(IntTy*)&arg2);
        }
        else
            return atomicCompareExchangeStrongNoResult!(succ, fail)(here, arg1, arg2);
    }

    /// Compare-and-set for shared value type
    bool cas(T, V1, V2)(shared(T)* here, V1 ifThis, V2 writeThis) pure nothrow @nogc @trusted
    if (!is(T == class) && (is(T : V1) || is(shared T : V1)))
    in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
    {
        static if (is (V1 == shared U1, U1))
            alias Thunk1 = U1;
        else
            alias Thunk1 = V1;
        static if (is (V2 == shared U2, U2))
            alias Thunk2 = U2;
        else
        {
            import core.internal.traits : hasUnsharedIndirections;
            static assert(!hasUnsharedIndirections!V2,
                          "Copying `" ~ V2.stringof ~ "* writeThis` to `" ~
                          shared(T).stringof ~ "* here` would violate shared.");
            alias Thunk2 = V2;
        }
        return cas(cast(T*)here, *cast(Thunk1*)&ifThis, *cast(Thunk2*)&writeThis);
    }

    /// Compare-and-set for `shared` reference type (`class`)
    bool cas(T, V1, V2)(shared(T)* here, shared(V1) ifThis, shared(V2) writeThis)
    pure nothrow @nogc @trusted
    if (is(T == class))
    in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
    {
        return atomicCompareExchangeStrongNoResult!(succ, fail)(
            cast(T*)here, cast(V1)ifThis, cast(V2)writeThis);
    }

    /// Compare-and-exchange for non-`shared` types
    bool cas(T, V)(T* here, T* ifThis, V writeThis) pure nothrow @nogc @trusted
    if (!is(T == shared) && !is(V == shared))
    in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
    {
        // resolve implicit conversions
        T arg1 = writeThis;

        static if (__traits(isFloating, T))
        {
            alias IntTy = IntForFloat!T;
            return atomicCompareExchangeStrong!(succ, fail)(
                cast(IntTy*)here, cast(IntTy*)ifThis, *cast(IntTy*)&writeThis);
        }
        else
            return atomicCompareExchangeStrong!(succ, fail)(here, ifThis, writeThis);
    }

    /// Compare and exchange for mixed-`shared`ness types
    bool cas(T, V1, V2)(shared(T)* here, V1* ifThis, V2 writeThis) pure nothrow @nogc @trusted
    if (!is(T == class) && (is(T : V1) || is(shared T : V1)))
    in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
    {
        static if (is (V1 == shared U1, U1))
            alias Thunk1 = U1;
        else
        {
            import core.internal.traits : hasUnsharedIndirections;
            static assert(!hasUnsharedIndirections!V1,
                          "Copying `" ~ shared(T).stringof ~ "* here` to `" ~
                          V1.stringof ~ "* ifThis` would violate shared.");
            alias Thunk1 = V1;
        }
        static if (is (V2 == shared U2, U2))
            alias Thunk2 = U2;
        else
        {
            import core.internal.traits : hasUnsharedIndirections;
            static assert(!hasUnsharedIndirections!V2,
                          "Copying `" ~ V2.stringof ~ "* writeThis` to `" ~
                          shared(T).stringof ~ "* here` would violate shared.");
            alias Thunk2 = V2;
        }
        static assert (is(T : Thunk1),
                       "Mismatching types for `here` and `ifThis`: `" ~
                       shared(T).stringof ~ "` and `" ~ V1.stringof ~ "`.");
        return cas(cast(T*)here, cast(Thunk1*)ifThis, *cast(Thunk2*)&writeThis);
    }

    /// Compare-and-exchange for `class`
    bool cas(T, V)(shared(T)* here, shared(T)* ifThis, shared(V) writeThis)
    pure nothrow @nogc @trusted
    if (is(T == class))
    in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
    {
        return atomicCompareExchangeStrong!(succ, fail)(
            cast(T*)here, cast(T*)ifThis, cast(V)writeThis);
    }
}

/**
* Stores 'writeThis' to the memory referenced by 'here' if the value
* referenced by 'here' is equal to 'ifThis'.
* The 'weak' version of cas may spuriously fail. It is recommended to
* use `casWeak` only when `cas` would be used in a loop.
* This operation is both
* lock-free and atomic.
*
* Params:
*  here      = The address of the destination variable.
*  writeThis = The value to store.
*  ifThis    = The comparison value.
*
* Returns:
*  true if the store occurred, false if not.
*/
bool casWeak(MemoryOrder succ = MemoryOrder.seq,MemoryOrder fail = MemoryOrder.seq,T,V1,V2)(T* here, V1 ifThis, V2 writeThis) pure nothrow @nogc @trusted
    if (!is(T == shared) && is(T : V1))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    // resolve implicit conversions
    T arg1 = ifThis;
    T arg2 = writeThis;

    static if (__traits(isFloating, T))
    {
        alias IntTy = IntForFloat!T;
        return atomicCompareExchangeWeakNoResult!(succ, fail)(cast(IntTy*)here, *cast(IntTy*)&arg1, *cast(IntTy*)&arg2);
    }
    else
        return atomicCompareExchangeWeakNoResult!(succ, fail)(here, arg1, arg2);
}

/// Ditto
bool casWeak(MemoryOrder succ = MemoryOrder.seq,MemoryOrder fail = MemoryOrder.seq,T,V1,V2)(shared(T)* here, V1 ifThis, V2 writeThis) pure nothrow @nogc @trusted
    if (!is(T == class) && (is(T : V1) || is(shared T : V1)))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    static if (is (V1 == shared U1, U1))
        alias Thunk1 = U1;
    else
        alias Thunk1 = V1;
    static if (is (V2 == shared U2, U2))
        alias Thunk2 = U2;
    else
    {
        import core.internal.traits : hasUnsharedIndirections;
        static assert(!hasUnsharedIndirections!V2, "Copying `" ~ V2.stringof ~ "* writeThis` to `" ~ shared(T).stringof ~ "* here` would violate shared.");
        alias Thunk2 = V2;
    }
    return casWeak!(succ, fail)(cast(T*)here, *cast(Thunk1*)&ifThis, *cast(Thunk2*)&writeThis);
}

/// Ditto
bool casWeak(MemoryOrder succ = MemoryOrder.seq,MemoryOrder fail = MemoryOrder.seq,T,V1,V2)(shared(T)* here, shared(V1) ifThis, shared(V2) writeThis) pure nothrow @nogc @trusted
    if (is(T == class))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    return atomicCompareExchangeWeakNoResult!(succ, fail)(cast(T*)here, cast(V1)ifThis, cast(V2)writeThis);
}

/**
* Stores 'writeThis' to the memory referenced by 'here' if the value
* referenced by 'here' is equal to the value referenced by 'ifThis'.
* The prior value referenced by 'here' is written to `ifThis` and
* returned to the user.
* The 'weak' version of cas may spuriously fail. It is recommended to
* use `casWeak` only when `cas` would be used in a loop.
* This operation is both lock-free and atomic.
*
* Params:
*  here      = The address of the destination variable.
*  writeThis = The value to store.
*  ifThis    = The address of the value to compare, and receives the prior value of `here` as output.
*
* Returns:
*  true if the store occurred, false if not.
*/
bool casWeak(MemoryOrder succ = MemoryOrder.seq,MemoryOrder fail = MemoryOrder.seq,T,V)(T* here, T* ifThis, V writeThis) pure nothrow @nogc @trusted
    if (!is(T == shared S, S) && !is(V == shared U, U))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    // resolve implicit conversions
    T arg1 = writeThis;

    static if (__traits(isFloating, T))
    {
        alias IntTy = IntForFloat!T;
        return atomicCompareExchangeWeak!(succ, fail)(cast(IntTy*)here, cast(IntTy*)ifThis, *cast(IntTy*)&writeThis);
    }
    else
        return atomicCompareExchangeWeak!(succ, fail)(here, ifThis, writeThis);
}

/// Ditto
bool casWeak(MemoryOrder succ = MemoryOrder.seq,MemoryOrder fail = MemoryOrder.seq,T,V1,V2)(shared(T)* here, V1* ifThis, V2 writeThis) pure nothrow @nogc @trusted
    if (!is(T == class) && (is(T : V1) || is(shared T : V1)))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    static if (is (V1 == shared U1, U1))
        alias Thunk1 = U1;
    else
    {
        import core.internal.traits : hasUnsharedIndirections;
        static assert(!hasUnsharedIndirections!V1, "Copying `" ~ shared(T).stringof ~ "* here` to `" ~ V1.stringof ~ "* ifThis` would violate shared.");
        alias Thunk1 = V1;
    }
    static if (is (V2 == shared U2, U2))
        alias Thunk2 = U2;
    else
    {
        import core.internal.traits : hasUnsharedIndirections;
        static assert(!hasUnsharedIndirections!V2, "Copying `" ~ V2.stringof ~ "* writeThis` to `" ~ shared(T).stringof ~ "* here` would violate shared.");
        alias Thunk2 = V2;
    }
    static assert (is(T : Thunk1), "Mismatching types for `here` and `ifThis`: `" ~ shared(T).stringof ~ "` and `" ~ V1.stringof ~ "`.");
    return casWeak!(succ, fail)(cast(T*)here, cast(Thunk1*)ifThis, *cast(Thunk2*)&writeThis);
}

/// Ditto
bool casWeak(MemoryOrder succ = MemoryOrder.seq,MemoryOrder fail = MemoryOrder.seq,T,V)(shared(T)* here, shared(T)* ifThis, shared(V) writeThis) pure nothrow @nogc @trusted
    if (is(T == class))
in (atomicPtrIsProperlyAligned(here), "Argument `here` is not properly aligned")
{
    return atomicCompareExchangeWeak!(succ, fail)(cast(T*)here, cast(T*)ifThis, cast(V)writeThis);
}

/**
 * Inserts a full load/store memory fence (on platforms that need it). This ensures
 * that all loads and stores before a call to this function are executed before any
 * loads and stores after the call.
 */
void atomicFence(MemoryOrder order = MemoryOrder.seq)() pure nothrow @nogc @safe
{
    core.internal.atomic.atomicFence!order();
}

/**
 * Gives a hint to the processor that the calling thread is in a 'spin-wait' loop,
 * allowing to more efficiently allocate resources.
 */
void pause() pure nothrow @nogc @safe
{
    core.internal.atomic.pause();
}

/**
 * Performs the binary operation 'op' on val using 'mod' as the modifier.
 *
 * Params:
 *  val = The target variable.
 *  mod = The modifier to apply.
 *
 * Returns:
 *  The result of the operation.
 */
TailShared!T atomicOp(string op, T, V1)(ref shared T val, V1 mod) pure nothrow @nogc @safe
    if (__traits(compiles, mixin("*cast(T*)&val" ~ op ~ "mod")))
in (atomicValueIsProperlyAligned(val))
{
    // binary operators
    //
    // +    -   *   /   %   ^^  &
    // |    ^   <<  >>  >>> ~   in
    // ==   !=  <   <=  >   >=
    static if (op == "+"  || op == "-"  || op == "*"  || op == "/"   ||
                op == "%"  || op == "^^" || op == "&"  || op == "|"   ||
                op == "^"  || op == "<<" || op == ">>" || op == ">>>" ||
                op == "~"  || // skip "in"
                op == "==" || op == "!=" || op == "<"  || op == "<="  ||
                op == ">"  || op == ">=")
    {
        T get = atomicLoad!(MemoryOrder.raw, T)(val);
        mixin("return get " ~ op ~ " mod;");
    }
    else
    // assignment operators
    //
    // +=   -=  *=  /=  %=  ^^= &=
    // |=   ^=  <<= >>= >>>=    ~=
    static if (op == "+=" && __traits(isIntegral, T) && __traits(isIntegral, V1) && T.sizeof <= size_t.sizeof && V1.sizeof <= size_t.sizeof)
    {
        return cast(T)(atomicFetchAdd(val, mod) + mod);
    }
    else static if (op == "-=" && __traits(isIntegral, T) && __traits(isIntegral, V1) && T.sizeof <= size_t.sizeof && V1.sizeof <= size_t.sizeof)
    {
        return cast(T)(atomicFetchSub(val, mod) - mod);
    }
    else static if (op == "+=" || op == "-="  || op == "*="  || op == "/=" ||
                op == "%=" || op == "^^=" || op == "&="  || op == "|=" ||
                op == "^=" || op == "<<=" || op == ">>=" || op == ">>>=") // skip "~="
    {
        T set, get = atomicLoad!(MemoryOrder.raw, T)(val);
        do
        {
            set = get;
            mixin("set " ~ op ~ " mod;");
        } while (!casWeakByRef(val, get, set));
        return set;
    }
    else
    {
        static assert(false, "Operation not supported.");
    }
}


version (D_InlineAsm_X86)
{
    enum has64BitXCHG = false;
    enum has64BitCAS = true;
    enum has128BitCAS = false;
}
else version (D_InlineAsm_X86_64)
{
    enum has64BitXCHG = true;
    enum has64BitCAS = true;
    enum has128BitCAS = true;
}
else version (GNU)
{
    import gcc.config;
    enum has64BitCAS = GNU_Have_64Bit_Atomics;
    enum has64BitXCHG = GNU_Have_64Bit_Atomics;
    enum has128BitCAS = GNU_Have_LibAtomic;
}
else
{
    enum has64BitXCHG = false;
    enum has64BitCAS = false;
    enum has128BitCAS = false;
}

private
{
    bool atomicValueIsProperlyAligned(T)(ref T val) pure nothrow @nogc @trusted
    {
        return atomicPtrIsProperlyAligned(&val);
    }

    bool atomicPtrIsProperlyAligned(T)(T* ptr) pure nothrow @nogc @safe
    {
        // NOTE: Strictly speaking, the x86 supports atomic operations on
        //       unaligned values.  However, this is far slower than the
        //       common case, so such behavior should be prohibited.
        static if (T.sizeof > size_t.sizeof)
        {
            version (X86)
            {
                // cmpxchg8b only requires 4-bytes alignment
                return cast(size_t)ptr % size_t.sizeof == 0;
            }
            else
            {
                // e.g., x86_64 cmpxchg16b requires 16-bytes alignment
                return cast(size_t)ptr % T.sizeof == 0;
            }
        }
        else
        {
            return cast(size_t)ptr % T.sizeof == 0;
        }
    }

    template IntForFloat(F)
        if (__traits(isFloating, F))
    {
        static if (F.sizeof == 4)
            alias IntForFloat = uint;
        else static if (F.sizeof == 8)
            alias IntForFloat = ulong;
        else
            static assert (false, "Invalid floating point type: " ~ F.stringof ~ ", only support `float` and `double`.");
    }

    template IntForStruct(S)
        if (is(S == struct))
    {
        static if (S.sizeof == 1)
            alias IntForFloat = ubyte;
        else static if (F.sizeof == 2)
            alias IntForFloat = ushort;
        else static if (F.sizeof == 4)
            alias IntForFloat = uint;
        else static if (F.sizeof == 8)
            alias IntForFloat = ulong;
        else static if (F.sizeof == 16)
            alias IntForFloat = ulong[2]; // TODO: what's the best type here? slice/delegates pass in registers...
        else
            static assert (ValidateStruct!S);
    }

    template ValidateStruct(S)
        if (is(S == struct))
    {
        import core.internal.traits : hasElaborateAssign;

        // `(x & (x-1)) == 0` checks that x is a power of 2.
        static assert (S.sizeof <= size_t.sizeof * 2
            && (S.sizeof & (S.sizeof - 1)) == 0,
            S.stringof ~ " has invalid size for atomic operations.");
        static assert (!hasElaborateAssign!S, S.stringof ~ " may not have an elaborate assignment when used with atomic operations.");

        enum ValidateStruct = true;
    }

    // TODO: it'd be nice if we had @trusted scopes; we could remove this...
    bool casWeakByRef(T,V1,V2)(ref T value, ref V1 ifThis, V2 writeThis) pure nothrow @nogc @trusted
    {
        return casWeak(&value, &ifThis, writeThis);
    }

    /* Construct a type with a shared tail, and if possible with an unshared
    head. */
    template TailShared(U) if (!is(U == shared))
    {
        alias TailShared = .TailShared!(shared U);
    }
    template TailShared(S) if (is(S == shared))
    {
        // Get the unshared variant of S.
        static if (is(S U == shared U)) {}
        else static assert(false, "Should never be triggered. The `static " ~
            "if` declares `U` as the unshared version of the shared type " ~
            "`S`. `S` is explicitly declared as shared, so getting `U` " ~
            "should always work.");

        static if (is(S : U))
            alias TailShared = U;
        else static if (is(S == struct))
        {
            enum implName = () {
                /* Start with "_impl". If S has a field with that name, append
                underscores until the clash is resolved. */
                string name = "_impl";
                string[] fieldNames;
                static foreach (alias field; S.tupleof)
                {
                    fieldNames ~= __traits(identifier, field);
                }
                static bool canFind(string[] haystack, string needle)
                {
                    foreach (candidate; haystack)
                    {
                        if (candidate == needle) return true;
                    }
                    return false;
                }
                while (canFind(fieldNames, name)) name ~= "_";
                return name;
            } ();
            struct TailShared
            {
                static foreach (i, alias field; S.tupleof)
                {
                    /* On @trusted: This is casting the field from shared(Foo)
                    to TailShared!Foo. The cast is safe because the field has
                    been loaded and is not shared anymore. */
                    mixin("
                        @trusted @property
                        ref " ~ __traits(identifier, field) ~ "()
                        {
                            alias R = TailShared!(typeof(field));
                            return * cast(R*) &" ~ implName ~ ".tupleof[i];
                        }
                    ");
                }
                mixin("
                    S " ~ implName ~ ";
                    alias " ~ implName ~ " this;
                ");
            }
        }
        else
            alias TailShared = S;
    }
    @safe unittest
    {
        // No tail (no indirections) -> fully unshared.

        static assert(is(TailShared!int == int));
        static assert(is(TailShared!(shared int) == int));

        static struct NoIndir { int i; }
        static assert(is(TailShared!NoIndir == NoIndir));
        static assert(is(TailShared!(shared NoIndir) == NoIndir));

        // Tail can be independently shared or is already -> tail-shared.

        static assert(is(TailShared!(int*) == shared(int)*));
        static assert(is(TailShared!(shared int*) == shared(int)*));
        static assert(is(TailShared!(shared(int)*) == shared(int)*));

        static assert(is(TailShared!(int[]) == shared(int)[]));
        static assert(is(TailShared!(shared int[]) == shared(int)[]));
        static assert(is(TailShared!(shared(int)[]) == shared(int)[]));

        static struct S1 { shared int* p; }
        static assert(is(TailShared!S1 == S1));
        static assert(is(TailShared!(shared S1) == S1));

        static struct S2 { shared(int)* p; }
        static assert(is(TailShared!S2 == S2));
        static assert(is(TailShared!(shared S2) == S2));

        // Tail follows shared-ness of head -> fully shared.

        static class C { int i; }
        static assert(is(TailShared!C == shared C));
        static assert(is(TailShared!(shared C) == shared C));

        /* However, structs get a wrapper that has getters which cast to
        TailShared. */

        static struct S3 { int* p; int _impl; int _impl_; int _impl__; }
        static assert(!is(TailShared!S3 : S3));
        static assert(is(TailShared!S3 : shared S3));
        static assert(is(TailShared!(shared S3) == TailShared!S3));

        static struct S4 { shared(int)** p; }
        static assert(!is(TailShared!S4 : S4));
        static assert(is(TailShared!S4 : shared S4));
        static assert(is(TailShared!(shared S4) == TailShared!S4));
    }
}


////////////////////////////////////////////////////////////////////////////////
// Unit Tests
////////////////////////////////////////////////////////////////////////////////


version (CoreUnittest)
{
    version (D_LP64)
    {
        enum hasDWCAS = has128BitCAS;
    }
    else
    {
        enum hasDWCAS = has64BitCAS;
    }

    void testXCHG(T)(T val) pure nothrow @nogc @trusted
    in
    {
        assert(val !is T.init);
    }
    do
    {
        T         base = cast(T)null;
        shared(T) atom = cast(shared(T))null;

        assert(base !is val, T.stringof);
        assert(atom is base, T.stringof);

        assert(atomicExchange(&atom, val) is base, T.stringof);
        assert(atom is val, T.stringof);
    }

    void testCAS(T)(T val) pure nothrow @nogc @trusted
    in
    {
        assert(val !is T.init);
    }
    do
    {
        T         base = cast(T)null;
        shared(T) atom = cast(shared(T))null;

        assert(base !is val, T.stringof);
        assert(atom is base, T.stringof);

        assert(cas(&atom, base, val), T.stringof);
        assert(atom is val, T.stringof);
        assert(!cas(&atom, base, base), T.stringof);
        assert(atom is val, T.stringof);

        atom = cast(shared(T))null;

        shared(T) arg = base;
        assert(cas(&atom, &arg, val), T.stringof);
        assert(arg is base, T.stringof);
        assert(atom is val, T.stringof);

        arg = base;
        assert(!cas(&atom, &arg, base), T.stringof);
        assert(arg is val, T.stringof);
        assert(atom is val, T.stringof);
    }

    void testLoadStore(MemoryOrder ms = MemoryOrder.seq, T)(T val = T.init + 1) pure nothrow @nogc @trusted
    {
        T         base = cast(T) 0;
        shared(T) atom = cast(T) 0;

        assert(base !is val);
        assert(atom is base);
        atomicStore!(ms)(atom, val);
        base = atomicLoad!(ms)(atom);

        assert(base is val, T.stringof);
        assert(atom is val);
    }


    void testType(T)(T val = T.init + 1) pure nothrow @nogc @safe
    {
        static if (T.sizeof < 8 || has64BitXCHG)
            testXCHG!(T)(val);
        testCAS!(T)(val);
        testLoadStore!(MemoryOrder.seq, T)(val);
        testLoadStore!(MemoryOrder.raw, T)(val);
    }

    @betterC @safe pure nothrow unittest
    {
        testType!(bool)();

        testType!(byte)();
        testType!(ubyte)();

        testType!(short)();
        testType!(ushort)();

        testType!(int)();
        testType!(uint)();
    }

    @safe pure nothrow unittest
    {

        testType!(shared int*)();

        static interface Inter {}
        static class KlassImpl : Inter {}
        testXCHG!(shared Inter)(new shared(KlassImpl));
        testCAS!(shared Inter)(new shared(KlassImpl));

        static class Klass {}
        testXCHG!(shared Klass)(new shared(Klass));
        testCAS!(shared Klass)(new shared(Klass));

        testXCHG!(shared int)(42);

        testType!(float)(0.1f);

        static if (has64BitCAS)
        {
            testType!(double)(0.1);
            testType!(long)();
            testType!(ulong)();
        }
        static if (has128BitCAS)
        {
            () @trusted
            {
                align(16) struct Big { long a, b; }

                shared(Big) atom;
                shared(Big) base;
                shared(Big) arg;
                shared(Big) val = Big(1, 2);

                assert(cas(&atom, arg, val), Big.stringof);
                assert(atom is val, Big.stringof);
                assert(!cas(&atom, arg, val), Big.stringof);
                assert(atom is val, Big.stringof);

                atom = Big();
                assert(cas(&atom, &arg, val), Big.stringof);
                assert(arg is base, Big.stringof);
                assert(atom is val, Big.stringof);

                arg = Big();
                assert(!cas(&atom, &arg, base), Big.stringof);
                assert(arg is val, Big.stringof);
                assert(atom is val, Big.stringof);
            }();
        }

        shared(size_t) i;

        atomicOp!"+="(i, cast(size_t) 1);
        assert(i == 1);

        atomicOp!"-="(i, cast(size_t) 1);
        assert(i == 0);

        shared float f = 0.1f;
        atomicOp!"+="(f, 0.1f);
        assert(f > 0.1999f && f < 0.2001f);

        static if (has64BitCAS)
        {
            shared double d = 0.1;
            atomicOp!"+="(d, 0.1);
            assert(d > 0.1999 && d < 0.2001);
        }
    }

    @betterC pure nothrow unittest
    {
        static if (has128BitCAS)
        {
            struct DoubleValue
            {
                long value1;
                long value2;
            }

            align(16) shared DoubleValue a;
            atomicStore(a, DoubleValue(1,2));
            assert(a.value1 == 1 && a.value2 ==2);

            while (!cas(&a, DoubleValue(1,2), DoubleValue(3,4))){}
            assert(a.value1 == 3 && a.value2 ==4);

            align(16) DoubleValue b = atomicLoad(a);
            assert(b.value1 == 3 && b.value2 ==4);
        }

        static if (hasDWCAS)
        {
            static struct List { size_t gen; List* next; }
            shared(List) head;
            assert(cas(&head, shared(List)(0, null), shared(List)(1, cast(List*)1)));
            assert(head.gen == 1);
            assert(cast(size_t)head.next == 1);
        }

        // https://issues.dlang.org/show_bug.cgi?id=20629
        static struct Struct
        {
            uint a, b;
        }
        shared Struct s1 = Struct(1, 2);
        atomicStore(s1, Struct(3, 4));
        assert(cast(uint) s1.a == 3);
        assert(cast(uint) s1.b == 4);
    }

    // https://issues.dlang.org/show_bug.cgi?id=20844
    static if (hasDWCAS)
    {
        debug: // tests CAS in-contract

        pure nothrow unittest
        {
            import core.exception : AssertError;

            align(16) shared ubyte[2 * size_t.sizeof + 1] data;
            auto misalignedPointer = cast(size_t[2]*) &data[1];
            size_t[2] x;

            try
                cas(misalignedPointer, x, x);
            catch (AssertError)
                return;

            assert(0, "should have failed");
        }
    }

    @betterC pure nothrow @nogc @safe unittest
    {
        int a;
        if (casWeak!(MemoryOrder.acq_rel, MemoryOrder.raw)(&a, 0, 4))
            assert(a == 4);
    }

    @betterC pure nothrow unittest
    {
        static struct S { int val; }
        auto s = shared(S)(1);

        shared(S*) ptr;

        // head unshared
        shared(S)* ifThis = null;
        shared(S)* writeThis = &s;
        assert(ptr is null);
        assert(cas(&ptr, ifThis, writeThis));
        assert(ptr is writeThis);

        // head shared
        shared(S*) ifThis2 = writeThis;
        shared(S*) writeThis2 = null;
        assert(cas(&ptr, ifThis2, writeThis2));
        assert(ptr is null);
    }

    unittest
    {
        import core.thread;

        // Use heap memory to ensure an optimizing
        // compiler doesn't put things in registers.
        uint* x = new uint();
        bool* f = new bool();
        uint* r = new uint();

        auto thr = new Thread(()
        {
            while (!*f)
            {
            }

            atomicFence();

            *r = *x;
        });

        thr.start();

        *x = 42;

        atomicFence();

        *f = true;

        atomicFence();

        thr.join();

        assert(*r == 42);
    }

    // === atomicFetchAdd and atomicFetchSub operations ====
    @betterC pure nothrow @nogc @safe unittest
    {
        shared ubyte u8 = 1;
        shared ushort u16 = 2;
        shared uint u32 = 3;
        shared byte i8 = 5;
        shared short i16 = 6;
        shared int i32 = 7;

        assert(atomicOp!"+="(u8, 8) == 9);
        assert(atomicOp!"+="(u16, 8) == 10);
        assert(atomicOp!"+="(u32, 8) == 11);
        assert(atomicOp!"+="(i8, 8) == 13);
        assert(atomicOp!"+="(i16, 8) == 14);
        assert(atomicOp!"+="(i32, 8) == 15);
        version (D_LP64)
        {
            shared ulong u64 = 4;
            shared long i64 = 8;
            assert(atomicOp!"+="(u64, 8) == 12);
            assert(atomicOp!"+="(i64, 8) == 16);
        }
    }

    @betterC pure nothrow @nogc unittest
    {
        byte[10] byteArray = [1, 3, 5, 7, 9, 11, 13, 15, 17, 19];
        ulong[10] ulongArray = [2, 4, 6, 8, 10, 12, 14, 16, 19, 20];

        {
            auto array = byteArray;
            byte* ptr = &array[0];
            byte* prevPtr = atomicFetchAdd(ptr, 3);
            assert(prevPtr == &array[0]);
            assert(*prevPtr == 1);
            assert(*ptr == 7);
        }
        {
            auto array = ulongArray;
            ulong* ptr = &array[0];
            ulong* prevPtr = atomicFetchAdd(ptr, 3);
            assert(prevPtr == &array[0]);
            assert(*prevPtr == 2);
            assert(*ptr == 8);
        }
    }

    @betterC pure nothrow @nogc @safe unittest
    {
        shared ubyte u8 = 1;
        shared ushort u16 = 2;
        shared uint u32 = 3;
        shared byte i8 = 5;
        shared short i16 = 6;
        shared int i32 = 7;

        assert(atomicOp!"-="(u8, 1) == 0);
        assert(atomicOp!"-="(u16, 1) == 1);
        assert(atomicOp!"-="(u32, 1) == 2);
        assert(atomicOp!"-="(i8, 1) == 4);
        assert(atomicOp!"-="(i16, 1) == 5);
        assert(atomicOp!"-="(i32, 1) == 6);
        version (D_LP64)
        {
            shared ulong u64 = 4;
            shared long i64 = 8;
            assert(atomicOp!"-="(u64, 1) == 3);
            assert(atomicOp!"-="(i64, 1) == 7);
        }
    }

    @betterC pure nothrow @nogc unittest
    {
        byte[10] byteArray = [1, 3, 5, 7, 9, 11, 13, 15, 17, 19];
        ulong[10] ulongArray = [2, 4, 6, 8, 10, 12, 14, 16, 19, 20];

        {
            auto array = byteArray;
            byte* ptr = &array[5];
            byte* prevPtr = atomicFetchSub(ptr, 4);
            assert(prevPtr == &array[5]);
            assert(*prevPtr == 11);
            assert(*ptr == 3); // https://issues.dlang.org/show_bug.cgi?id=21578
        }
        {
            auto array = ulongArray;
            ulong* ptr = &array[5];
            ulong* prevPtr = atomicFetchSub(ptr, 4);
            assert(prevPtr == &array[5]);
            assert(*prevPtr == 12);
            assert(*ptr == 4); // https://issues.dlang.org/show_bug.cgi?id=21578
        }
    }

    @betterC pure nothrow @nogc @safe unittest // issue 16651
    {
        shared ulong a = 2;
        uint b = 1;
        atomicOp!"-="(a, b);
        assert(a == 1);

        shared uint c = 2;
        ubyte d = 1;
        atomicOp!"-="(c, d);
        assert(c == 1);
    }

    pure nothrow @safe unittest // issue 16230
    {
        shared int i;
        static assert(is(typeof(atomicLoad(i)) == int));

        shared int* p;
        static assert(is(typeof(atomicLoad(p)) == shared(int)*));

        shared int[] a;
        static if (__traits(compiles, atomicLoad(a)))
        {
            static assert(is(typeof(atomicLoad(a)) == shared(int)[]));
        }

        static struct S { int* _impl; }
        shared S s;
        static assert(is(typeof(atomicLoad(s)) : shared S));
        static assert(is(typeof(atomicLoad(s)._impl) == shared(int)*));
        auto u = atomicLoad(s);
        assert(u._impl is null);
        u._impl = new shared int(42);
        assert(atomicLoad(*u._impl) == 42);

        static struct S2 { S s; }
        shared S2 s2;
        static assert(is(typeof(atomicLoad(s2).s) == TailShared!S));

        static struct S3 { size_t head; int* tail; }
        shared S3 s3;
        static if (__traits(compiles, atomicLoad(s3)))
        {
            static assert(is(typeof(atomicLoad(s3).head) == size_t));
            static assert(is(typeof(atomicLoad(s3).tail) == shared(int)*));
        }

        static class C { int i; }
        shared C c;
        static assert(is(typeof(atomicLoad(c)) == shared C));

        static struct NoIndirections { int i; }
        shared NoIndirections n;
        static assert(is(typeof(atomicLoad(n)) == NoIndirections));
    }
}