1 /* Copyright (C) 2012-2018 Free Software Foundation, Inc. 2 Contributed by Richard Henderson <rth@redhat.com>. 3 4 This file is part of the GNU Atomic Library (libatomic). 5 6 Libatomic is free software; you can redistribute it and/or modify it 7 under the terms of the GNU General Public License as published by 8 the Free Software Foundation; either version 3, or (at your option) 9 any later version. 10 11 Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY 12 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 13 FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 more details. 15 16 Under Section 7 of GPL version 3, you are granted additional 17 permissions described in the GCC Runtime Library Exception, version 18 3.1, as published by the Free Software Foundation. 19 20 You should have received a copy of the GNU General Public License and 21 a copy of the GCC Runtime Library Exception along with this program; 22 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 <http://www.gnu.org/licenses/>. */ 24 25 /* This file contains data types and function declarations that are 26 private to the implementation of libatomic. */ 27 28 #ifndef LIBATOMIC_H 29 #define LIBATOMIC_H 1 30 31 #include "auto-config.h" 32 #include <stdbool.h> 33 #include <stdint.h> 34 #include <stddef.h> 35 #include <limits.h> 36 #include <string.h> 37 38 39 /* Symbol concatenation macros. */ 40 #define C2_(X,Y) X ## Y 41 #define C2(X,Y) C2_(X,Y) 42 #define C3_(X,Y,Z) X ## Y ## Z 43 #define C3(X,Y,Z) C3_(X,Y,Z) 44 #define C4_(W,X,Y,Z) W ## X ## Y ## Z 45 #define C4(W,X,Y,Z) C4_(W,X,Y,Z) 46 47 /* Stringification macros. */ 48 #define S2(X) #X 49 #define S(X) S2(X) 50 51 /* All of the primitive types on which we operate. */ 52 typedef unsigned U_1 __attribute__((mode(QI))); 53 #if HAVE_INT2 54 typedef unsigned U_2 __attribute__((mode(HI))); 55 #endif 56 #if HAVE_INT4 57 typedef unsigned U_4 __attribute__((mode(SI))); 58 #endif 59 #if HAVE_INT8 60 typedef unsigned U_8 __attribute__((mode(DI))); 61 #endif 62 #if HAVE_INT16 63 typedef unsigned U_16 __attribute__((mode(TI))); 64 #endif 65 66 /* The widest type that we support. */ 67 #if HAVE_INT16 68 # define MAX_SIZE 16 69 #elif HAVE_INT8 70 # define MAX_SIZE 8 71 #elif HAVE_INT4 72 # define MAX_SIZE 4 73 #elif HAVE_INT2 74 # define MAX_SIZE 2 75 #else 76 # define MAX_SIZE 1 77 #endif 78 typedef C2(U_,MAX_SIZE) U_MAX; 79 80 /* Provide dummy fallback types so that stuff is syntactically correct 81 without having to overdo the ifdefs. The code using these should 82 always be protected with the HAVE_INT{n} macros. */ 83 #if !HAVE_INT2 84 typedef U_MAX U_2; 85 #endif 86 #if !HAVE_INT4 87 typedef U_MAX U_4; 88 #endif 89 #if !HAVE_INT8 90 typedef U_MAX U_8; 91 #endif 92 #if !HAVE_INT16 93 typedef U_MAX U_16; 94 #endif 95 96 union max_size_u 97 { 98 U_1 b[MAX_SIZE]; 99 U_2 i2; 100 U_4 i4; 101 U_8 i8; 102 U_16 i16; 103 }; 104 105 /* The "word" size of the machine. */ 106 typedef unsigned UWORD __attribute__((mode(word))); 107 108 /* Macros for handing sub-word sized quantities. */ 109 #define MASK_1 ((UWORD)0xff) 110 #define MASK_2 ((UWORD)0xffff) 111 #define MASK_4 ((UWORD)0xffffffff) 112 #define INVERT_MASK_1 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT)) 113 #define INVERT_MASK_2 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT)) 114 #define INVERT_MASK_4 ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT)) 115 116 /* Most of the files in this library are compiled multiple times with 117 N defined to be a power of 2 between 1 and 16. The SIZE macro is 118 then used to append _N to the symbol being manipulated. */ 119 #define SIZE(X) C3(X,_,N) 120 #define WSIZE(X) C3(X,_,WORDSIZE) 121 #define PTR(N,X) ((C2(U_,N) *)X) 122 123 /* And thus, the type on which this compilation will be operating. */ 124 #define ITYPE SIZE(I) 125 #define UTYPE SIZE(U) 126 127 /* Utility macros for GCC attributes. */ 128 #define UNUSED __attribute__((unused)) 129 #ifdef HAVE_ATTRIBUTE_VISIBILITY 130 # define HIDDEN __attribute__((visibility("hidden"))) 131 #else 132 # define HIDDEN 133 #endif 134 135 /* Occasionally we have to play games with internal and external symbol 136 names, in order to work around builtin functions of the same name. 137 This macro sets the external name of the function appropriately. */ 138 #define ASMNAME(X) __asm__(S(C2(__USER_LABEL_PREFIX__,X))) 139 140 /* Locking for a "small" operation. In the bare-metal single processor 141 cases this could be implemented by disabling interrupts. Thus the extra 142 word passed between the two functions, saving the interrupt level. 143 It is assumed that the object being locked does not cross the locking 144 granularity. 145 146 Not actually declared here so that they can be defined static inline 147 in a target-specfic <host-config.h>. 148 149 UWORD protect_start (void *ptr); 150 void protect_end (void *ptr, UWORD); 151 */ 152 153 /* Locking for a "large' operation. This should always be some sort of 154 test-and-set operation, as we assume that the interrupt latency would 155 be unreasonably large. */ 156 void libat_lock_n (void *ptr, size_t n); 157 void libat_unlock_n (void *ptr, size_t n); 158 159 /* We'll need to declare all of the sized functions a few times... */ 160 #define DECLARE_ALL_SIZED(N) DECLARE_ALL_SIZED_(N,C2(U_,N)) 161 #define DECLARE_ALL_SIZED_(N,T) \ 162 DECLARE_1(T, C2(load_,N), (T *mptr, int)); \ 163 DECLARE_1(void, C2(store_,N), (T *mptr, T val, int)); \ 164 DECLARE_1(T, C2(exchange_,N), (T *mptr, T, int)); \ 165 DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \ 166 DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int)); \ 167 DECLARE_1(T, C2(fetch_add_,N), (T *mptr, T, int)); \ 168 DECLARE_1(T, C2(fetch_sub_,N), (T *mptr, T, int)); \ 169 DECLARE_1(T, C2(fetch_and_,N), (T *mptr, T, int)); \ 170 DECLARE_1(T, C2(fetch_xor_,N), (T *mptr, T, int)); \ 171 DECLARE_1(T, C2(fetch_or_,N), (T *mptr, T, int)); \ 172 DECLARE_1(T, C2(fetch_nand_,N), (T *mptr, T, int)); \ 173 DECLARE_1(T, C2(add_fetch_,N), (T *mptr, T, int)); \ 174 DECLARE_1(T, C2(sub_fetch_,N), (T *mptr, T, int)); \ 175 DECLARE_1(T, C2(and_fetch_,N), (T *mptr, T, int)); \ 176 DECLARE_1(T, C2(xor_fetch_,N), (T *mptr, T, int)); \ 177 DECLARE_1(T, C2(or_fetch_,N), (T *mptr, T, int)); \ 178 DECLARE_1(T, C2(nand_fetch_,N), (T *mptr, T, int)) 179 180 /* All sized operations are implemented in hidden functions prefixed with 181 "libat_". These are either renamed or aliased to the expected prefix 182 of "__atomic". Some amount of renaming is required to avoid hiding or 183 conflicting with the builtins of the same name, but this additional 184 use of hidden symbols (where appropriate) avoids unnecessary PLT entries 185 on relevant targets. */ 186 187 #if IFUNC_ALT 188 # define MAN(X) ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN 189 #elif defined(HAVE_ATTRIBUTE_ALIAS) 190 # define MAN(X) HIDDEN 191 #else 192 # define MAN(X) ASMNAME(C2(__atomic_,X)) 193 #endif 194 195 #if !defined(N) && HAVE_IFUNC 196 # define DECLARE_1(RET,NAME,ARGS) \ 197 RET C2(libat_,NAME) ARGS MAN(NAME); \ 198 RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME)) 199 #else 200 # define DECLARE_1(RET,NAME,ARGS) RET C2(libat_,NAME) ARGS MAN(NAME) 201 #endif 202 203 /* Prefix to use when calling internal, possibly ifunc'ed functions. */ 204 #if HAVE_IFUNC 205 # define local_ ifunc_ 206 #else 207 # define local_ libat_ 208 #endif 209 210 DECLARE_ALL_SIZED(1); 211 DECLARE_ALL_SIZED(2); 212 DECLARE_ALL_SIZED(4); 213 DECLARE_ALL_SIZED(8); 214 DECLARE_ALL_SIZED(16); 215 216 #undef DECLARE_1 217 #undef DECLARE_ALL_SIZED 218 #undef DECLARE_ALL_SIZED_ 219 220 /* And the generic sized versions. */ 221 void libat_load (size_t, void *, void *, int) MAN(load); 222 void libat_store (size_t, void *, void *, int) MAN(store); 223 void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange); 224 bool libat_compare_exchange (size_t, void *, void *, void *, int, int) 225 MAN(compare_exchange); 226 bool libat_is_lock_free (size_t, void *) MAN(is_lock_free); 227 228 #undef MAN 229 230 #include <host-config.h> 231 232 /* We don't have IFUNC_NCOND until after host-config.h. */ 233 #if !HAVE_IFUNC 234 # define IFUNC_NCOND(N) 0 235 #endif 236 237 #if IFUNC_ALT 238 # define EXPORT_ALIAS(X) /* exported symbol in non-alternate file */ 239 #elif defined(N) && IFUNC_NCOND(N) 240 # if IFUNC_NCOND(N) == 1 241 # define GEN_SELECTOR(X) \ 242 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \ 243 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \ 244 { \ 245 if (IFUNC_COND_1) \ 246 return C3(libat_,X,_i1); \ 247 return C2(libat_,X); \ 248 } 249 # elif IFUNC_NCOND(N) == 2 250 # define GEN_SELECTOR(X) \ 251 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \ 252 extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \ 253 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \ 254 { \ 255 if (IFUNC_COND_1) \ 256 return C3(libat_,X,_i1); \ 257 if (IFUNC_COND_2) \ 258 return C3(libat_,X,_i2); \ 259 return C2(libat_,X); \ 260 } 261 # elif IFUNC_NCOND(N) == 3 262 # define GEN_SELECTOR(X) \ 263 extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN; \ 264 extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN; \ 265 extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN; \ 266 static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \ 267 { \ 268 if (IFUNC_COND_1) \ 269 return C3(libat_,X,_i1); \ 270 if (IFUNC_COND_2) \ 271 return C3(libat_,X,_i2); \ 272 if (IFUNC_COND_3) \ 273 return C3(libat_,X,_i3); \ 274 return C2(libat_,X); \ 275 } 276 # else 277 # error "Unsupported number of ifunc alternatives." 278 # endif 279 # define EXPORT_ALIAS(X) \ 280 GEN_SELECTOR(X) \ 281 typeof(C2(libat_,X)) C2(ifunc_,X) \ 282 ASMNAME(C2(__atomic_,X)) \ 283 __attribute__((ifunc(S(C2(select_,X))))) 284 #elif defined(HAVE_ATTRIBUTE_ALIAS) 285 # define EXPORT_ALIAS(X) \ 286 extern typeof(C2(libat_,X)) C2(export_,X) \ 287 ASMNAME(C2(__atomic_,X)) \ 288 __attribute__((alias(S(C2(libat_,X))))) 289 #else 290 # define EXPORT_ALIAS(X) /* original symbol is exported */ 291 #endif 292 293 #endif /* LIBATOMIC_H */ 294