12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
209d4e0edSPaul Mackerras /*
309d4e0edSPaul Mackerras * Generic implementation of 64-bit atomics using spinlocks,
409d4e0edSPaul Mackerras * useful on processors that don't have 64-bit atomic instructions.
509d4e0edSPaul Mackerras *
609d4e0edSPaul Mackerras * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
709d4e0edSPaul Mackerras */
809d4e0edSPaul Mackerras #include <linux/types.h>
909d4e0edSPaul Mackerras #include <linux/cache.h>
1009d4e0edSPaul Mackerras #include <linux/spinlock.h>
1109d4e0edSPaul Mackerras #include <linux/init.h>
128bc3bcc9SPaul Gortmaker #include <linux/export.h>
1360063497SArun Sharma #include <linux/atomic.h>
1409d4e0edSPaul Mackerras
1509d4e0edSPaul Mackerras /*
1609d4e0edSPaul Mackerras * We use a hashed array of spinlocks to provide exclusive access
1709d4e0edSPaul Mackerras * to each atomic64_t variable. Since this is expected to used on
1809d4e0edSPaul Mackerras * systems with small numbers of CPUs (<= 4 or so), we use a
1909d4e0edSPaul Mackerras * relatively small array of 16 spinlocks to avoid wasting too much
2009d4e0edSPaul Mackerras * memory on the spinlock array.
2109d4e0edSPaul Mackerras */
2209d4e0edSPaul Mackerras #define NR_LOCKS 16
2309d4e0edSPaul Mackerras
2409d4e0edSPaul Mackerras /*
2509d4e0edSPaul Mackerras * Ensure each lock is in a separate cacheline.
2609d4e0edSPaul Mackerras */
2709d4e0edSPaul Mackerras static union {
28f59ca058SShan Hai raw_spinlock_t lock;
2909d4e0edSPaul Mackerras char pad[L1_CACHE_BYTES];
30fcc16882SStephen Boyd } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = {
31fcc16882SStephen Boyd [0 ... (NR_LOCKS - 1)] = {
32fcc16882SStephen Boyd .lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock),
33fcc16882SStephen Boyd },
34fcc16882SStephen Boyd };
3509d4e0edSPaul Mackerras
lock_addr(const atomic64_t * v)36cb475de3SYong Zhang static inline raw_spinlock_t *lock_addr(const atomic64_t *v)
3709d4e0edSPaul Mackerras {
3809d4e0edSPaul Mackerras unsigned long addr = (unsigned long) v;
3909d4e0edSPaul Mackerras
4009d4e0edSPaul Mackerras addr >>= L1_CACHE_SHIFT;
4109d4e0edSPaul Mackerras addr ^= (addr >> 8) ^ (addr >> 16);
4209d4e0edSPaul Mackerras return &atomic64_lock[addr & (NR_LOCKS - 1)].lock;
4309d4e0edSPaul Mackerras }
4409d4e0edSPaul Mackerras
generic_atomic64_read(const atomic64_t * v)45*1bdadf46SMark Rutland s64 generic_atomic64_read(const atomic64_t *v)
4609d4e0edSPaul Mackerras {
4709d4e0edSPaul Mackerras unsigned long flags;
48cb475de3SYong Zhang raw_spinlock_t *lock = lock_addr(v);
499255813dSMark Rutland s64 val;
5009d4e0edSPaul Mackerras
51f59ca058SShan Hai raw_spin_lock_irqsave(lock, flags);
5209d4e0edSPaul Mackerras val = v->counter;
53f59ca058SShan Hai raw_spin_unlock_irqrestore(lock, flags);
5409d4e0edSPaul Mackerras return val;
5509d4e0edSPaul Mackerras }
56*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_read);
5709d4e0edSPaul Mackerras
generic_atomic64_set(atomic64_t * v,s64 i)58*1bdadf46SMark Rutland void generic_atomic64_set(atomic64_t *v, s64 i)
5909d4e0edSPaul Mackerras {
6009d4e0edSPaul Mackerras unsigned long flags;
61cb475de3SYong Zhang raw_spinlock_t *lock = lock_addr(v);
6209d4e0edSPaul Mackerras
63f59ca058SShan Hai raw_spin_lock_irqsave(lock, flags);
6409d4e0edSPaul Mackerras v->counter = i;
65f59ca058SShan Hai raw_spin_unlock_irqrestore(lock, flags);
6609d4e0edSPaul Mackerras }
67*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_set);
6809d4e0edSPaul Mackerras
69560cb12aSPeter Zijlstra #define ATOMIC64_OP(op, c_op) \
70*1bdadf46SMark Rutland void generic_atomic64_##op(s64 a, atomic64_t *v) \
71560cb12aSPeter Zijlstra { \
72560cb12aSPeter Zijlstra unsigned long flags; \
73560cb12aSPeter Zijlstra raw_spinlock_t *lock = lock_addr(v); \
74560cb12aSPeter Zijlstra \
75560cb12aSPeter Zijlstra raw_spin_lock_irqsave(lock, flags); \
76560cb12aSPeter Zijlstra v->counter c_op a; \
77560cb12aSPeter Zijlstra raw_spin_unlock_irqrestore(lock, flags); \
78560cb12aSPeter Zijlstra } \
79*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_##op);
8009d4e0edSPaul Mackerras
81560cb12aSPeter Zijlstra #define ATOMIC64_OP_RETURN(op, c_op) \
82*1bdadf46SMark Rutland s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \
83560cb12aSPeter Zijlstra { \
84560cb12aSPeter Zijlstra unsigned long flags; \
85560cb12aSPeter Zijlstra raw_spinlock_t *lock = lock_addr(v); \
869255813dSMark Rutland s64 val; \
87560cb12aSPeter Zijlstra \
88560cb12aSPeter Zijlstra raw_spin_lock_irqsave(lock, flags); \
89560cb12aSPeter Zijlstra val = (v->counter c_op a); \
90560cb12aSPeter Zijlstra raw_spin_unlock_irqrestore(lock, flags); \
91560cb12aSPeter Zijlstra return val; \
92560cb12aSPeter Zijlstra } \
93*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_##op##_return);
9409d4e0edSPaul Mackerras
9528aa2bdaSPeter Zijlstra #define ATOMIC64_FETCH_OP(op, c_op) \
96*1bdadf46SMark Rutland s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \
9728aa2bdaSPeter Zijlstra { \
9828aa2bdaSPeter Zijlstra unsigned long flags; \
9928aa2bdaSPeter Zijlstra raw_spinlock_t *lock = lock_addr(v); \
1009255813dSMark Rutland s64 val; \
10128aa2bdaSPeter Zijlstra \
10228aa2bdaSPeter Zijlstra raw_spin_lock_irqsave(lock, flags); \
10328aa2bdaSPeter Zijlstra val = v->counter; \
10428aa2bdaSPeter Zijlstra v->counter c_op a; \
10528aa2bdaSPeter Zijlstra raw_spin_unlock_irqrestore(lock, flags); \
10628aa2bdaSPeter Zijlstra return val; \
10728aa2bdaSPeter Zijlstra } \
108*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_fetch_##op);
10928aa2bdaSPeter Zijlstra
110560cb12aSPeter Zijlstra #define ATOMIC64_OPS(op, c_op) \
111560cb12aSPeter Zijlstra ATOMIC64_OP(op, c_op) \
11228aa2bdaSPeter Zijlstra ATOMIC64_OP_RETURN(op, c_op) \
11328aa2bdaSPeter Zijlstra ATOMIC64_FETCH_OP(op, c_op)
11409d4e0edSPaul Mackerras
115560cb12aSPeter Zijlstra ATOMIC64_OPS(add, +=)
116560cb12aSPeter Zijlstra ATOMIC64_OPS(sub, -=)
11709d4e0edSPaul Mackerras
118560cb12aSPeter Zijlstra #undef ATOMIC64_OPS
11928aa2bdaSPeter Zijlstra #define ATOMIC64_OPS(op, c_op) \
12028aa2bdaSPeter Zijlstra ATOMIC64_OP(op, c_op) \
12128aa2bdaSPeter Zijlstra ATOMIC64_FETCH_OP(op, c_op)
12228aa2bdaSPeter Zijlstra
12328aa2bdaSPeter Zijlstra ATOMIC64_OPS(and, &=)
12428aa2bdaSPeter Zijlstra ATOMIC64_OPS(or, |=)
12528aa2bdaSPeter Zijlstra ATOMIC64_OPS(xor, ^=)
12628aa2bdaSPeter Zijlstra
12728aa2bdaSPeter Zijlstra #undef ATOMIC64_OPS
12828aa2bdaSPeter Zijlstra #undef ATOMIC64_FETCH_OP
129560cb12aSPeter Zijlstra #undef ATOMIC64_OP
13009d4e0edSPaul Mackerras
generic_atomic64_dec_if_positive(atomic64_t * v)131*1bdadf46SMark Rutland s64 generic_atomic64_dec_if_positive(atomic64_t *v)
13209d4e0edSPaul Mackerras {
13309d4e0edSPaul Mackerras unsigned long flags;
134cb475de3SYong Zhang raw_spinlock_t *lock = lock_addr(v);
1359255813dSMark Rutland s64 val;
13609d4e0edSPaul Mackerras
137f59ca058SShan Hai raw_spin_lock_irqsave(lock, flags);
13809d4e0edSPaul Mackerras val = v->counter - 1;
13909d4e0edSPaul Mackerras if (val >= 0)
14009d4e0edSPaul Mackerras v->counter = val;
141f59ca058SShan Hai raw_spin_unlock_irqrestore(lock, flags);
14209d4e0edSPaul Mackerras return val;
14309d4e0edSPaul Mackerras }
144*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_dec_if_positive);
14509d4e0edSPaul Mackerras
generic_atomic64_cmpxchg(atomic64_t * v,s64 o,s64 n)146*1bdadf46SMark Rutland s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n)
14709d4e0edSPaul Mackerras {
14809d4e0edSPaul Mackerras unsigned long flags;
149cb475de3SYong Zhang raw_spinlock_t *lock = lock_addr(v);
1509255813dSMark Rutland s64 val;
15109d4e0edSPaul Mackerras
152f59ca058SShan Hai raw_spin_lock_irqsave(lock, flags);
15309d4e0edSPaul Mackerras val = v->counter;
15409d4e0edSPaul Mackerras if (val == o)
15509d4e0edSPaul Mackerras v->counter = n;
156f59ca058SShan Hai raw_spin_unlock_irqrestore(lock, flags);
15709d4e0edSPaul Mackerras return val;
15809d4e0edSPaul Mackerras }
159*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_cmpxchg);
16009d4e0edSPaul Mackerras
generic_atomic64_xchg(atomic64_t * v,s64 new)161*1bdadf46SMark Rutland s64 generic_atomic64_xchg(atomic64_t *v, s64 new)
16209d4e0edSPaul Mackerras {
16309d4e0edSPaul Mackerras unsigned long flags;
164cb475de3SYong Zhang raw_spinlock_t *lock = lock_addr(v);
1659255813dSMark Rutland s64 val;
16609d4e0edSPaul Mackerras
167f59ca058SShan Hai raw_spin_lock_irqsave(lock, flags);
16809d4e0edSPaul Mackerras val = v->counter;
16909d4e0edSPaul Mackerras v->counter = new;
170f59ca058SShan Hai raw_spin_unlock_irqrestore(lock, flags);
17109d4e0edSPaul Mackerras return val;
17209d4e0edSPaul Mackerras }
173*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_xchg);
17409d4e0edSPaul Mackerras
generic_atomic64_fetch_add_unless(atomic64_t * v,s64 a,s64 u)175*1bdadf46SMark Rutland s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
17609d4e0edSPaul Mackerras {
17709d4e0edSPaul Mackerras unsigned long flags;
178cb475de3SYong Zhang raw_spinlock_t *lock = lock_addr(v);
1799255813dSMark Rutland s64 val;
18009d4e0edSPaul Mackerras
181f59ca058SShan Hai raw_spin_lock_irqsave(lock, flags);
18200b808abSMark Rutland val = v->counter;
18300b808abSMark Rutland if (val != u)
18409d4e0edSPaul Mackerras v->counter += a;
185f59ca058SShan Hai raw_spin_unlock_irqrestore(lock, flags);
18600b808abSMark Rutland
18700b808abSMark Rutland return val;
18809d4e0edSPaul Mackerras }
189*1bdadf46SMark Rutland EXPORT_SYMBOL(generic_atomic64_fetch_add_unless);
190