1 /* 2 * Atomic operations on 64-bit quantities. 3 * 4 * Copyright (C) 2017 Red Hat, Inc. 5 * 6 * Author: Paolo Bonzini <pbonzini@redhat.com> 7 * 8 * This work is licensed under the terms of the GNU GPL, version 2 or later. 9 * See the COPYING file in the top-level directory. 10 */ 11 12 #ifndef QEMU_STATS64_H 13 #define QEMU_STATS64_H 14 15 #include "qemu/atomic.h" 16 17 /* This provides atomic operations on 64-bit type, using a reader-writer 18 * spinlock on architectures that do not have 64-bit accesses. Even on 19 * those architectures, it tries hard not to take the lock. 20 */ 21 22 typedef struct Stat64 { 23 #ifdef CONFIG_ATOMIC64 24 uint64_t value; 25 #else 26 uint32_t low, high; 27 uint32_t lock; 28 #endif 29 } Stat64; 30 31 #ifdef CONFIG_ATOMIC64 32 static inline void stat64_init(Stat64 *s, uint64_t value) 33 { 34 /* This is not guaranteed to be atomic! */ 35 *s = (Stat64) { value }; 36 } 37 38 static inline uint64_t stat64_get(const Stat64 *s) 39 { 40 return atomic_read__nocheck(&s->value); 41 } 42 43 static inline void stat64_add(Stat64 *s, uint64_t value) 44 { 45 atomic_add(&s->value, value); 46 } 47 48 static inline void stat64_min(Stat64 *s, uint64_t value) 49 { 50 uint64_t orig = atomic_read__nocheck(&s->value); 51 while (orig > value) { 52 orig = atomic_cmpxchg__nocheck(&s->value, orig, value); 53 } 54 } 55 56 static inline void stat64_max(Stat64 *s, uint64_t value) 57 { 58 uint64_t orig = atomic_read__nocheck(&s->value); 59 while (orig < value) { 60 orig = atomic_cmpxchg__nocheck(&s->value, orig, value); 61 } 62 } 63 #else 64 uint64_t stat64_get(const Stat64 *s); 65 bool stat64_min_slow(Stat64 *s, uint64_t value); 66 bool stat64_max_slow(Stat64 *s, uint64_t value); 67 bool stat64_add32_carry(Stat64 *s, uint32_t low, uint32_t high); 68 69 static inline void stat64_init(Stat64 *s, uint64_t value) 70 { 71 /* This is not guaranteed to be atomic! */ 72 *s = (Stat64) { .low = value, .high = value >> 32, .lock = 0 }; 73 } 74 75 static inline void stat64_add(Stat64 *s, uint64_t value) 76 { 77 uint32_t low, high; 78 high = value >> 32; 79 low = (uint32_t) value; 80 if (!low) { 81 if (high) { 82 atomic_add(&s->high, high); 83 } 84 return; 85 } 86 87 for (;;) { 88 uint32_t orig = s->low; 89 uint32_t result = orig + low; 90 uint32_t old; 91 92 if (result < low || high) { 93 /* If the high part is affected, take the lock. */ 94 if (stat64_add32_carry(s, low, high)) { 95 return; 96 } 97 continue; 98 } 99 100 /* No carry, try with a 32-bit cmpxchg. The result is independent of 101 * the high 32 bits, so it can race just fine with stat64_add32_carry 102 * and even stat64_get! 103 */ 104 old = atomic_cmpxchg(&s->low, orig, result); 105 if (orig == old) { 106 return; 107 } 108 } 109 } 110 111 static inline void stat64_min(Stat64 *s, uint64_t value) 112 { 113 uint32_t low, high; 114 uint32_t orig_low, orig_high; 115 116 high = value >> 32; 117 low = (uint32_t) value; 118 do { 119 orig_high = atomic_read(&s->high); 120 if (orig_high < high) { 121 return; 122 } 123 124 if (orig_high == high) { 125 /* High 32 bits are equal. Read low after high, otherwise we 126 * can get a false positive (e.g. 0x1235,0x0000 changes to 127 * 0x1234,0x8000 and we read it as 0x1234,0x0000). Pairs with 128 * the write barrier in stat64_min_slow. 129 */ 130 smp_rmb(); 131 orig_low = atomic_read(&s->low); 132 if (orig_low <= low) { 133 return; 134 } 135 136 /* See if we were lucky and a writer raced against us. The 137 * barrier is theoretically unnecessary, but if we remove it 138 * we may miss being lucky. 139 */ 140 smp_rmb(); 141 orig_high = atomic_read(&s->high); 142 if (orig_high < high) { 143 return; 144 } 145 } 146 147 /* If the value changes in any way, we have to take the lock. */ 148 } while (!stat64_min_slow(s, value)); 149 } 150 151 static inline void stat64_max(Stat64 *s, uint64_t value) 152 { 153 uint32_t low, high; 154 uint32_t orig_low, orig_high; 155 156 high = value >> 32; 157 low = (uint32_t) value; 158 do { 159 orig_high = atomic_read(&s->high); 160 if (orig_high > high) { 161 return; 162 } 163 164 if (orig_high == high) { 165 /* High 32 bits are equal. Read low after high, otherwise we 166 * can get a false positive (e.g. 0x1234,0x8000 changes to 167 * 0x1235,0x0000 and we read it as 0x1235,0x8000). Pairs with 168 * the write barrier in stat64_max_slow. 169 */ 170 smp_rmb(); 171 orig_low = atomic_read(&s->low); 172 if (orig_low >= low) { 173 return; 174 } 175 176 /* See if we were lucky and a writer raced against us. The 177 * barrier is theoretically unnecessary, but if we remove it 178 * we may miss being lucky. 179 */ 180 smp_rmb(); 181 orig_high = atomic_read(&s->high); 182 if (orig_high > high) { 183 return; 184 } 185 } 186 187 /* If the value changes in any way, we have to take the lock. */ 188 } while (!stat64_max_slow(s, value)); 189 } 190 191 #endif 192 193 #endif 194