1 /* 2 * Copyright (c) 2013-2017 Intel Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #ifndef _OFI_ATOM_H_ 34 #define _OFI_ATOM_H_ 35 36 #include "config.h" 37 38 #include <assert.h> 39 #include <pthread.h> 40 #include <stdlib.h> 41 42 #include <ofi_lock.h> 43 #include <ofi_osd.h> 44 45 #ifdef HAVE_ATOMICS 46 # include <stdatomic.h> 47 #endif 48 49 50 #ifdef __cplusplus 51 extern "C" { 52 #endif 53 54 55 #if ENABLE_DEBUG 56 #define ATOMIC_DEF_INIT int is_initialized 57 #define ATOMIC_IS_INITIALIZED(atomic) assert(atomic->is_initialized) 58 #define ATOMIC_INIT(atomic) atomic->is_initialized = 1 59 #else 60 #define ATOMIC_DEF_INIT 61 #define ATOMIC_IS_INITIALIZED(atomic) 62 #define ATOMIC_INIT(atomic) 63 #endif 64 65 #ifdef HAVE_ATOMICS 66 #ifdef HAVE_ATOMICS_LEAST_TYPES 67 typedef atomic_int_least32_t ofi_atomic_int32_t; 68 typedef atomic_int_least64_t ofi_atomic_int64_t; 69 #else 70 typedef atomic_int ofi_atomic_int32_t; 71 typedef atomic_long ofi_atomic_int64_t; 72 #endif 73 74 #define OFI_ATOMIC_DEFINE(radix) \ 75 typedef struct { \ 76 ofi_atomic_int##radix##_t val; \ 77 ATOMIC_DEF_INIT; \ 78 } ofi_atomic##radix##_t; \ 79 \ 80 static inline \ 81 int##radix##_t ofi_atomic_inc##radix(ofi_atomic##radix##_t *atomic) \ 82 { \ 83 ATOMIC_IS_INITIALIZED(atomic); \ 84 return (int##radix##_t)atomic_fetch_add_explicit(&atomic->val, 1, \ 85 memory_order_acq_rel) + 1; \ 86 } \ 87 static inline \ 88 int##radix##_t ofi_atomic_dec##radix(ofi_atomic##radix##_t *atomic) \ 89 { \ 90 ATOMIC_IS_INITIALIZED(atomic); \ 91 return (int##radix##_t)atomic_fetch_sub_explicit(&atomic->val, 1, \ 92 memory_order_acq_rel) - 1; \ 93 } \ 94 static inline \ 95 int##radix##_t ofi_atomic_set##radix(ofi_atomic##radix##_t *atomic, int##radix##_t value) \ 96 { \ 97 ATOMIC_IS_INITIALIZED(atomic); \ 98 atomic_store(&atomic->val, value); \ 99 return (int##radix##_t)value; \ 100 } \ 101 static inline \ 102 int##radix##_t ofi_atomic_get##radix(ofi_atomic##radix##_t *atomic) \ 103 { \ 104 ATOMIC_IS_INITIALIZED(atomic); \ 105 return (int##radix##_t)atomic_load(&atomic->val); \ 106 } \ 107 static inline \ 108 void ofi_atomic_initialize##radix(ofi_atomic##radix##_t *atomic, int##radix##_t value) \ 109 { \ 110 atomic_init(&atomic->val, value); \ 111 ATOMIC_INIT(atomic); \ 112 } \ 113 static inline \ 114 int##radix##_t ofi_atomic_add##radix(ofi_atomic##radix##_t *atomic, int##radix##_t val) \ 115 { \ 116 ATOMIC_IS_INITIALIZED(atomic); \ 117 return (int##radix##_t)atomic_fetch_add_explicit(&atomic->val, val, \ 118 memory_order_acq_rel) + val; \ 119 } \ 120 static inline \ 121 int##radix##_t ofi_atomic_sub##radix(ofi_atomic##radix##_t *atomic, int##radix##_t val) \ 122 { \ 123 ATOMIC_IS_INITIALIZED(atomic); \ 124 return (int##radix##_t)atomic_fetch_sub_explicit(&atomic->val, val, \ 125 memory_order_acq_rel) - val; \ 126 } 127 128 #elif defined HAVE_BUILTIN_ATOMICS 129 # if ENABLE_DEBUG 130 # define ATOMIC_T(radix) \ 131 struct { \ 132 int##radix##_t val; \ 133 ATOMIC_DEF_INIT; \ 134 } 135 136 # define ofi_atomic_ptr(atomic) (&((atomic)->val)) 137 # else 138 # define ATOMIC_T(radix) int##radix##_t 139 # define ofi_atomic_ptr(atomic) (atomic) 140 # endif 141 142 #define OFI_ATOMIC_DEFINE(radix) \ 143 typedef ATOMIC_T(radix) ofi_atomic##radix##_t; \ 144 \ 145 static inline \ 146 int##radix##_t ofi_atomic_add##radix(ofi_atomic##radix##_t *atomic, int##radix##_t val) \ 147 { \ 148 ATOMIC_IS_INITIALIZED(atomic); \ 149 return (int##radix##_t)ofi_atomic_add_and_fetch(radix, ofi_atomic_ptr(atomic), val); \ 150 } \ 151 static inline \ 152 int##radix##_t ofi_atomic_sub##radix(ofi_atomic##radix##_t *atomic, int##radix##_t val) \ 153 { \ 154 ATOMIC_IS_INITIALIZED(atomic); \ 155 return (int##radix##_t)ofi_atomic_sub_and_fetch(radix, ofi_atomic_ptr(atomic), val); \ 156 } \ 157 static inline \ 158 int##radix##_t ofi_atomic_inc##radix(ofi_atomic##radix##_t *atomic) \ 159 { \ 160 ATOMIC_IS_INITIALIZED(atomic); \ 161 return ofi_atomic_add##radix(atomic, 1); \ 162 } \ 163 static inline \ 164 int##radix##_t ofi_atomic_dec##radix(ofi_atomic##radix##_t *atomic) \ 165 { \ 166 ATOMIC_IS_INITIALIZED(atomic); \ 167 return ofi_atomic_sub##radix(atomic, 1); \ 168 } \ 169 static inline \ 170 int##radix##_t ofi_atomic_set##radix(ofi_atomic##radix##_t *atomic, int##radix##_t value) \ 171 { \ 172 ATOMIC_IS_INITIALIZED(atomic); \ 173 *(ofi_atomic_ptr(atomic)) = value; \ 174 return value; \ 175 } \ 176 static inline \ 177 int##radix##_t ofi_atomic_get##radix(ofi_atomic##radix##_t *atomic) \ 178 { \ 179 ATOMIC_IS_INITIALIZED(atomic); \ 180 return *ofi_atomic_ptr(atomic); \ 181 } \ 182 static inline \ 183 void ofi_atomic_initialize##radix(ofi_atomic##radix##_t *atomic, int##radix##_t value) \ 184 { \ 185 *(ofi_atomic_ptr(atomic)) = value; \ 186 ATOMIC_INIT(atomic); \ 187 } 188 189 #else /* HAVE_ATOMICS */ 190 191 #define OFI_ATOMIC_DEFINE(radix) \ 192 typedef struct { \ 193 fastlock_t lock; \ 194 int##radix##_t val; \ 195 ATOMIC_DEF_INIT; \ 196 } ofi_atomic##radix##_t; \ 197 \ 198 static inline \ 199 int##radix##_t ofi_atomic_inc##radix(ofi_atomic##radix##_t *atomic) \ 200 { \ 201 int##radix##_t v = 0; \ 202 ATOMIC_IS_INITIALIZED(atomic); \ 203 fastlock_acquire(&atomic->lock); \ 204 v = ++(atomic->val); \ 205 fastlock_release(&atomic->lock); \ 206 return v; \ 207 } \ 208 static inline \ 209 int##radix##_t ofi_atomic_dec##radix(ofi_atomic##radix##_t *atomic) \ 210 { \ 211 int##radix##_t v = 0; \ 212 ATOMIC_IS_INITIALIZED(atomic); \ 213 fastlock_acquire(&atomic->lock); \ 214 v = --(atomic->val); \ 215 fastlock_release(&atomic->lock); \ 216 return v; \ 217 } \ 218 static inline \ 219 int##radix##_t ofi_atomic_set##radix(ofi_atomic##radix##_t *atomic, \ 220 int##radix##_t value) \ 221 { \ 222 ATOMIC_IS_INITIALIZED(atomic); \ 223 fastlock_acquire(&atomic->lock); \ 224 atomic->val = value; \ 225 fastlock_release(&atomic->lock); \ 226 return value; \ 227 } \ 228 static inline int##radix##_t ofi_atomic_get##radix(ofi_atomic##radix##_t *atomic) \ 229 { \ 230 ATOMIC_IS_INITIALIZED(atomic); \ 231 return atomic->val; \ 232 } \ 233 static inline \ 234 void ofi_atomic_initialize##radix(ofi_atomic##radix##_t *atomic, \ 235 int##radix##_t value) \ 236 { \ 237 fastlock_init(&atomic->lock); \ 238 atomic->val = value; \ 239 ATOMIC_INIT(atomic); \ 240 } \ 241 static inline \ 242 int##radix##_t ofi_atomic_add##radix(ofi_atomic##radix##_t *atomic, \ 243 int##radix##_t val) \ 244 { \ 245 int##radix##_t v; \ 246 ATOMIC_IS_INITIALIZED(atomic); \ 247 fastlock_acquire(&atomic->lock); \ 248 atomic->val += val; \ 249 v = atomic->val; \ 250 fastlock_release(&atomic->lock); \ 251 return v; \ 252 } \ 253 static inline \ 254 int##radix##_t ofi_atomic_sub##radix(ofi_atomic##radix##_t *atomic, \ 255 int##radix##_t val) \ 256 { \ 257 int##radix##_t v; \ 258 ATOMIC_IS_INITIALIZED(atomic); \ 259 fastlock_acquire(&atomic->lock); \ 260 atomic->val -= val; \ 261 v = atomic->val; \ 262 fastlock_release(&atomic->lock); \ 263 return v; \ 264 } 265 #endif // HAVE_ATOMICS 266 267 OFI_ATOMIC_DEFINE(32) 268 OFI_ATOMIC_DEFINE(64) 269 270 #ifdef __cplusplus 271 } 272 #endif 273 274 #endif /* _OFI_ATOM_H_ */ 275