1 /* 2 * %CopyrightBegin% 3 * 4 * Copyright Ericsson AB 2010-2015. All Rights Reserved. 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * %CopyrightEnd% 19 */ 20 21 /* 22 * Description: Native atomic ethread support when using gcc's __atomic 23 * and __sync builtins 24 * Author: Rickard Green 25 */ 26 27 #if !defined(ETHREAD_GCC_NATIVE_H__) && ETHR_GCC_COMPILER 28 #define ETHREAD_GCC_NATIVE_H__ 29 30 #ifndef ETHR_MEMBAR 31 # include "ethr_membar.h" 32 #endif 33 34 #define ETHR_GCC_VERSIONS_MASK__ 28 35 36 #undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ 37 #undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ 38 #undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ 39 #undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ 40 #undef ETHR_GCC_RELAXED_VERSIONS__ 41 #undef ETHR_GCC_RELAXED_MOD_VERSIONS__ 42 #undef ETHR_GCC_ACQB_VERSIONS__ 43 #undef ETHR_GCC_ACQB_MOD_VERSIONS__ 44 #undef ETHR_GCC_RELB_VERSIONS__ 45 #undef ETHR_GCC_RELB_MOD_VERSIONS__ 46 #undef ETHR_GCC_MB_MOD_VERSIONS__ 47 #undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 48 49 #define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ \ 50 ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS 51 52 /* 53 * True GNU GCCs before version 4.8 do not emit a memory barrier 54 * after the load in the __atomic_load_n(_, __ATOMIC_ACQUIRE) 55 * case (which is needed on most architectures). 56 */ 57 #undef ETHR___atomic_load_ACQUIRE_barrier_bug 58 #if ETHR_GCC_COMPILER != ETHR_GCC_COMPILER_TRUE 59 60 #if ETHR_GCC_COMPILER == ETHR_GCC_COMPILER_CLANG \ 61 && defined(__apple_build_version__) \ 62 && __clang_major__ >= 12 63 /* Apples clang verified not to have this bug */ 64 # define ETHR___atomic_load_ACQUIRE_barrier_bug 0 65 /* Also trust builtin barriers */ 66 # undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 67 # define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 1 68 # else 69 /* 70 * Another gcc compatible compiler. We have no information 71 * about the existence of this bug, but we assume 72 * that it is not impossible that it could have 73 * been "inherited". Therefore, until we are certain 74 * that the bug does not exist, we assume that it 75 * does. 76 */ 77 # define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ 78 # endif 79 80 #elif !ETHR_AT_LEAST_GCC_VSN__(4, 8, 0) 81 /* True gcc of version < 4.8, i.e., bug exist... */ 82 # define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ 83 #elif ETHR_AT_LEAST_GCC_VSN__(8, 3, 0) \ 84 && (defined(__arm64__) || defined(__aarch64__) || defined(__arm__)) \ 85 && ETHR_SIZEOF_PTR == 8 86 /* Verified not to have this bug */ 87 # define ETHR___atomic_load_ACQUIRE_barrier_bug 0 88 /* Also trust builtin barriers */ 89 # undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 90 # define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 1 91 #elif ETHR_AT_LEAST_GCC_VSN__(9, 3, 0) \ 92 && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)) \ 93 && ETHR_SIZEOF_PTR == 8 94 /* Verified not to have this bug */ 95 # define ETHR___atomic_load_ACQUIRE_barrier_bug 0 96 /* Also trust builtin barriers */ 97 # undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 98 # define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 1 99 #else /* True gcc of version >= 4.8 */ 100 /* 101 * Sizes less than or equal to word size have been fixed, 102 * but double word size has not been fixed. 103 */ 104 # if ETHR_SIZEOF_PTR == 8 105 # define ETHR___atomic_load_ACQUIRE_barrier_bug \ 106 (~(8|4) & ETHR_GCC_VERSIONS_MASK__) 107 # elif ETHR_SIZEOF_PTR == 4 108 # define ETHR___atomic_load_ACQUIRE_barrier_bug \ 109 (~4 & ETHR_GCC_VERSIONS_MASK__) 110 # else 111 # error word size not supported 112 # endif 113 #endif 114 115 #define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ 0 116 #define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ 0 117 #define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ 0 118 #define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ 0 119 #define ETHR_GCC_RELAXED_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 120 #define ETHR_GCC_RELAXED_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 121 122 #if ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 123 # define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 124 # define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 125 # define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 126 # define ETHR_GCC_RELB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 127 #else 128 /* 129 * This is currently the default (on most platforms) since 130 * we've seen too many memory barrier bugs produced by gcc... 131 */ 132 # define ETHR_GCC_ACQB_VERSIONS__ 0 133 # define ETHR_GCC_ACQB_MOD_VERSIONS__ 0 134 # define ETHR_GCC_RELB_VERSIONS__ 0 135 # define ETHR_GCC_RELB_MOD_VERSIONS__ 0 136 #endif 137 /* 138 * In the general case we do not want any full barrier versions 139 * if we can implement more relaxed ones (using __atomic_* builtins). 140 * This since the implementations normally need extra memory barrier 141 * instructions to implement these. The x86/x86_64 implementations 142 * are an exception see below. 143 */ 144 #define ETHR_GCC_MB_MOD_VERSIONS__ \ 145 (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___atomic_compare_exchange_n) 146 147 #if ETHR_SIZEOF_PTR == 8 148 # define ETHR_GCC_VOLATILE_BIT_MASK__ 12 149 #elif ETHR_SIZEOF_PTR == 4 150 # define ETHR_GCC_VOLATILE_BIT_MASK__ 4 151 #endif 152 153 #if defined(__i386__) || defined(__x86_64__) || defined(__sparc__) \ 154 || defined(__powerpc__) || defined(__ppc__) || defined(__mips__) \ 155 || defined(__alpha__) || defined(__ia64__) 156 157 /* 158 * Aligned volatile stores and loads of data smaller 159 * than or equal to word size are atomic... 160 */ 161 # undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ 162 # define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ ETHR_GCC_VOLATILE_BIT_MASK__ 163 # undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ 164 # define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ ETHR_GCC_VOLATILE_BIT_MASK__ 165 166 #elif defined(__arm__) 167 168 /* volatile stores are problematic on some machines... */ 169 # undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ 170 # define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ ETHR_GCC_VOLATILE_BIT_MASK__ 171 172 #endif 173 174 #if defined(__ia64__) 175 176 /* Volatile stores produce stores with release barriers. */ 177 # undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ 178 # define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ ETHR_GCC_VOLATILE_BIT_MASK__ 179 180 /* Volatile loads produce loads with acquire barrier. */ 181 # undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ 182 # define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ ETHR_GCC_VOLATILE_BIT_MASK__ 183 184 /* 185 * We trust gcc to produce acquire/release barriers on itanium. 186 * Since all atomic ops also have at least acquire or release 187 * barriers (also when passed the relaxed memory model) it 188 * would be very inefficient not to use these as native 189 * barriers on Itanium. 190 */ 191 # undef ETHR_GCC_ACQB_VERSIONS__ 192 # define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 193 # undef ETHR_GCC_ACQB_MOD_VERSIONS__ 194 # define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 195 # undef ETHR_GCC_RELB_VERSIONS__ 196 # define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 197 # undef ETHR_GCC_RELB_MOD_VERSIONS__ 198 # define ETHR_GCC_RELB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ 199 200 /* 201 * Itanium is not effected by the load acquire 202 * bug since the barrier is part of the instruction 203 * on Itanium (ld.acq), and not a separate instruction 204 * as on most platforms. 205 */ 206 # undef ETHR___atomic_load_ACQUIRE_barrier_bug 207 # define ETHR___atomic_load_ACQUIRE_barrier_bug 0 208 209 /* 210 * No point exposing relaxed versions since they are 211 * implemended using either acquire or release 212 * barriers. 213 */ 214 # undef ETHR_GCC_RELAXED_VERSIONS__ 215 # define ETHR_GCC_RELAXED_VERSIONS__ 0 216 217 /* #endif defined(__ia64__) */ 218 #elif defined(__i386__) || defined(__x86_64__) 219 220 /* 221 * Want full barrier versions of all modification 222 * operations since all of these are implemented 223 * using locked instructions implying full memory 224 * barriers. 225 */ 226 # undef ETHR_GCC_MB_MOD_VERSIONS__ 227 # define ETHR_GCC_MB_MOD_VERSIONS__ ETHR_HAVE___sync_val_compare_and_swap 228 229 /* 230 * No point exposing acquire/release versions 231 * when we got full memory barrier versions 232 * of modification operations since all of these 233 * are implemented using locked instructions 234 * implying full memory barriers. 235 */ 236 # if ETHR_GCC_ACQB_MOD_VERSIONS__ 237 # undef ETHR_GCC_ACQB_MOD_VERSIONS__ 238 # define ETHR_GCC_ACQB_MOD_VERSIONS__ \ 239 (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___sync_val_compare_and_swap) 240 # endif 241 # if ETHR_GCC_RELB_MOD_VERSIONS__ 242 # undef ETHR_GCC_RELB_MOD_VERSIONS__ 243 # define ETHR_GCC_RELB_MOD_VERSIONS__ \ 244 (ETHR_GCC_VERSIONS_MASK__ & ~ETHR_HAVE___sync_val_compare_and_swap) 245 # endif 246 247 # ifdef ETHR_X86_OUT_OF_ORDER 248 249 /* See above... */ 250 # undef ETHR_GCC_RELAXED_MOD_VERSIONS__ 251 # define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0 252 253 # else /* !ETHR_X86_OUT_OF_ORDER, i.e., we don't use any x86-OOO instructions... */ 254 255 /* 256 * Not effected by the load acquire barrier bug, 257 * since no barrier at all is needed for a load 258 * acquire... 259 */ 260 # undef ETHR___atomic_load_ACQUIRE_barrier_bug 261 # define ETHR___atomic_load_ACQUIRE_barrier_bug 0 262 263 /* Stores imply release barriers semantics. */ 264 # undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ 265 # define ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ ETHR_GCC_VOLATILE_BIT_MASK__ 266 267 /* Loads imply acquire barrier semantics. */ 268 # undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ 269 # define ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ ETHR_GCC_VOLATILE_BIT_MASK__ 270 271 /* 272 * Trust load acquire and store release for sizes 273 * where volatile operation implies these barrier 274 * semantics since no barriers are needed. 275 */ 276 # if !ETHR_GCC_ACQB_VERSIONS__ 277 # undef ETHR_GCC_ACQB_VERSIONS__ 278 # define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VOLATILE_BIT_MASK__ 279 # endif 280 # if !ETHR_GCC_RELB_VERSIONS__ 281 # undef ETHR_GCC_RELB_VERSIONS__ 282 # define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VOLATILE_BIT_MASK__ 283 # endif 284 285 /* 286 * No point exposing relaxed versions at all since 287 * all mod operations are implemented with locked 288 * instructions implying full memory barriers and 289 * volatile store and load imply release and 290 * acquire barrier semantics. 291 */ 292 # undef ETHR_GCC_RELAXED_VERSIONS__ 293 # define ETHR_GCC_RELAXED_VERSIONS__ 0 294 295 # endif /* !ETHR_X86_OUT_OF_ORDER */ 296 297 /* #endif defined(__i386__) || defined(__x86_64__) */ 298 #elif defined(__powerpc__) || defined(__ppc__) 299 300 # if !defined(ETHR_PPC_HAVE_LWSYNC) 301 /* 302 * Release barriers are typically implemented using 303 * the lwsync instruction. We want our runtime 304 * configure test to determine if the lwsync 305 * instruction is available on the system or not 306 * before we use it. Therefore, do not implement any 307 * native ops using the __ATOMIC_RELEASE model. 308 */ 309 # undef ETHR_GCC_RELB_VERSIONS__ 310 # define ETHR_GCC_RELB_VERSIONS__ 0 311 # if defined(ETHR_GCC_IMPLEMENT_ACQB_USING_LWSYNC) 312 /* 313 * Acquire barriers are usually implemented by other means 314 * than lwsync, but can be implemented using lwsync. Define 315 * ETHR_GCC_IMPLEMENT_ACQB_USING_LWSYNC if acquire barriers 316 * are implemented using lwsync. 317 */ 318 # undef ETHR_GCC_ACQB_VERSIONS__ 319 # define ETHR_GCC_ACQB_VERSIONS__ 0 320 # endif 321 # endif 322 323 #endif /* defined(__powerpc__) || defined(__ppc__) */ 324 325 #if !ETHR_GCC_RELAXED_VERSIONS__ 326 # undef ETHR_GCC_RELAXED_MOD_VERSIONS__ 327 # define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0 328 #endif 329 330 #if !ETHR_GCC_ACQB_VERSIONS__ 331 # undef ETHR_GCC_ACQB_MOD_VERSIONS__ 332 # define ETHR_GCC_ACQB_MOD_VERSIONS__ 0 333 #endif 334 335 #if !ETHR_GCC_RELB_VERSIONS__ 336 # undef ETHR_GCC_RELB_MOD_VERSIONS__ 337 # define ETHR_GCC_RELB_MOD_VERSIONS__ 0 338 #endif 339 340 #if !defined(ETHR_HAVE_NATIVE_ATOMIC32) 341 # define ETHR_ATOMIC_WANT_32BIT_IMPL__ 342 # include "ethr_atomic.h" 343 #endif 344 345 #if ETHR_SIZEOF_PTR == 8 && !defined(ETHR_HAVE_NATIVE_ATOMIC64) 346 # define ETHR_ATOMIC_WANT_64BIT_IMPL__ 347 # include "ethr_atomic.h" 348 #endif 349 350 #if defined(__x86_64__) 351 /* 352 * No instructions available for native implementation 353 * of these for dw-atomics... 354 */ 355 # undef ETHR_GCC_RELAXED_VERSIONS__ 356 # define ETHR_GCC_RELAXED_VERSIONS__ 0 357 # undef ETHR_GCC_ACQB_VERSIONS__ 358 # define ETHR_GCC_ACQB_VERSIONS__ 0 359 # undef ETHR_GCC_RELB_VERSIONS__ 360 # define ETHR_GCC_RELB_VERSIONS__ 0 361 #endif 362 363 #if !ETHR_GCC_RELAXED_VERSIONS__ 364 # undef ETHR_GCC_RELAXED_MOD_VERSIONS__ 365 # define ETHR_GCC_RELAXED_MOD_VERSIONS__ 0 366 #endif 367 368 #if !ETHR_GCC_ACQB_VERSIONS__ 369 # undef ETHR_GCC_ACQB_MOD_VERSIONS__ 370 # define ETHR_GCC_ACQB_MOD_VERSIONS__ 0 371 #endif 372 373 #if !ETHR_GCC_RELB_VERSIONS__ 374 # undef ETHR_GCC_RELB_MOD_VERSIONS__ 375 # define ETHR_GCC_RELB_MOD_VERSIONS__ 0 376 #endif 377 378 #if (!defined(ETHR_HAVE_NATIVE_DW_ATOMIC) \ 379 && !(ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64)) \ 380 && !(ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC128))) 381 # include "ethr_dw_atomic.h" 382 #endif 383 384 #undef ETHR___atomic_load_ACQUIRE_barrier_bug 385 #undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE__ 386 #undef ETHR_GCC_VOLATILE_STORE_IS_ATOMIC_STORE_RELB__ 387 #undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD__ 388 #undef ETHR_GCC_VOLATILE_LOAD_IS_ATOMIC_LOAD_ACQB__ 389 #undef ETHR_GCC_RELAXED_VERSIONS__ 390 #undef ETHR_GCC_RELB_VERSIONS__ 391 #undef ETHR_GCC_RELB_VERSIONS__ 392 #undef ETHR_GCC_RELAXED_MOD_VERSIONS__ 393 #undef ETHR_GCC_ACQB_MOD_VERSIONS__ 394 #undef ETHR_GCC_RELB_MOD_VERSIONS__ 395 #undef ETHR_GCC_MB_MOD_VERSIONS__ 396 397 #endif /* ETHREAD_GCC_NATIVE_H__ */ 398