1/* 2 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a copy 5 * of this software and associated documentation files (the "Software"), to deal 6 * in the Software without restriction, including without limitation the rights 7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 * copies of the Software, and to permit persons to whom the Software is 9 * furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 * SOFTWARE. 21 */ 22 23/* XSIZE_compare_and_swap (based on fetch_compare_and_swap) */ 24#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_full) \ 25 && !defined(AO_HAVE_XSIZE_compare_and_swap_full) 26 AO_INLINE int 27 AO_XSIZE_compare_and_swap_full(volatile XCTYPE *addr, XCTYPE old_val, 28 XCTYPE new_val) 29 { 30 return AO_XSIZE_fetch_compare_and_swap_full(addr, old_val, new_val) 31 == old_val; 32 } 33# define AO_HAVE_XSIZE_compare_and_swap_full 34#endif 35 36#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire) \ 37 && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire) 38 AO_INLINE int 39 AO_XSIZE_compare_and_swap_acquire(volatile XCTYPE *addr, XCTYPE old_val, 40 XCTYPE new_val) 41 { 42 return AO_XSIZE_fetch_compare_and_swap_acquire(addr, old_val, new_val) 43 == old_val; 44 } 45# define AO_HAVE_XSIZE_compare_and_swap_acquire 46#endif 47 48#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release) \ 49 && !defined(AO_HAVE_XSIZE_compare_and_swap_release) 50 AO_INLINE int 51 AO_XSIZE_compare_and_swap_release(volatile XCTYPE *addr, XCTYPE old_val, 52 XCTYPE new_val) 53 { 54 return AO_XSIZE_fetch_compare_and_swap_release(addr, old_val, new_val) 55 == old_val; 56 } 57# define AO_HAVE_XSIZE_compare_and_swap_release 58#endif 59 60#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_write) \ 61 && !defined(AO_HAVE_XSIZE_compare_and_swap_write) 62 AO_INLINE int 63 AO_XSIZE_compare_and_swap_write(volatile XCTYPE *addr, XCTYPE old_val, 64 XCTYPE new_val) 65 { 66 return AO_XSIZE_fetch_compare_and_swap_write(addr, old_val, new_val) 67 == old_val; 68 } 69# define AO_HAVE_XSIZE_compare_and_swap_write 70#endif 71 72#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_read) \ 73 && !defined(AO_HAVE_XSIZE_compare_and_swap_read) 74 AO_INLINE int 75 AO_XSIZE_compare_and_swap_read(volatile XCTYPE *addr, XCTYPE old_val, 76 XCTYPE new_val) 77 { 78 return AO_XSIZE_fetch_compare_and_swap_read(addr, old_val, new_val) 79 == old_val; 80 } 81# define AO_HAVE_XSIZE_compare_and_swap_read 82#endif 83 84#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap) \ 85 && !defined(AO_HAVE_XSIZE_compare_and_swap) 86 AO_INLINE int 87 AO_XSIZE_compare_and_swap(volatile XCTYPE *addr, XCTYPE old_val, 88 XCTYPE new_val) 89 { 90 return AO_XSIZE_fetch_compare_and_swap(addr, old_val, new_val) == old_val; 91 } 92# define AO_HAVE_XSIZE_compare_and_swap 93#endif 94 95#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_release_write) \ 96 && !defined(AO_HAVE_XSIZE_compare_and_swap_release_write) 97 AO_INLINE int 98 AO_XSIZE_compare_and_swap_release_write(volatile XCTYPE *addr, 99 XCTYPE old_val, XCTYPE new_val) 100 { 101 return AO_XSIZE_fetch_compare_and_swap_release_write(addr, old_val, 102 new_val) == old_val; 103 } 104# define AO_HAVE_XSIZE_compare_and_swap_release_write 105#endif 106 107#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_acquire_read) \ 108 && !defined(AO_HAVE_XSIZE_compare_and_swap_acquire_read) 109 AO_INLINE int 110 AO_XSIZE_compare_and_swap_acquire_read(volatile XCTYPE *addr, 111 XCTYPE old_val, XCTYPE new_val) 112 { 113 return AO_XSIZE_fetch_compare_and_swap_acquire_read(addr, old_val, 114 new_val) == old_val; 115 } 116# define AO_HAVE_XSIZE_compare_and_swap_acquire_read 117#endif 118 119#if defined(AO_HAVE_XSIZE_fetch_compare_and_swap_dd_acquire_read) \ 120 && !defined(AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read) 121 AO_INLINE int 122 AO_XSIZE_compare_and_swap_dd_acquire_read(volatile XCTYPE *addr, 123 XCTYPE old_val, XCTYPE new_val) 124 { 125 return AO_XSIZE_fetch_compare_and_swap_dd_acquire_read(addr, old_val, 126 new_val) == old_val; 127 } 128# define AO_HAVE_XSIZE_compare_and_swap_dd_acquire_read 129#endif 130 131/* XSIZE_fetch_and_add */ 132/* We first try to implement fetch_and_add variants in terms of the */ 133/* corresponding compare_and_swap variants to minimize adding barriers. */ 134#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ 135 && !defined(AO_HAVE_XSIZE_fetch_and_add_full) 136 AO_INLINE XCTYPE 137 AO_XSIZE_fetch_and_add_full(volatile XCTYPE *addr, XCTYPE incr) 138 { 139 XCTYPE old; 140 141 do 142 { 143 old = *(XCTYPE *)addr; 144 } 145 while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, 146 old + incr))); 147 return old; 148 } 149# define AO_HAVE_XSIZE_fetch_and_add_full 150#endif 151 152#if defined(AO_HAVE_XSIZE_compare_and_swap_acquire) \ 153 && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire) 154 AO_INLINE XCTYPE 155 AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr) 156 { 157 XCTYPE old; 158 159 do 160 { 161 old = *(XCTYPE *)addr; 162 } 163 while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_acquire(addr, old, 164 old + incr))); 165 return old; 166 } 167# define AO_HAVE_XSIZE_fetch_and_add_acquire 168#endif 169 170#if defined(AO_HAVE_XSIZE_compare_and_swap_release) \ 171 && !defined(AO_HAVE_XSIZE_fetch_and_add_release) 172 AO_INLINE XCTYPE 173 AO_XSIZE_fetch_and_add_release(volatile XCTYPE *addr, XCTYPE incr) 174 { 175 XCTYPE old; 176 177 do 178 { 179 old = *(XCTYPE *)addr; 180 } 181 while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_release(addr, old, 182 old + incr))); 183 return old; 184 } 185# define AO_HAVE_XSIZE_fetch_and_add_release 186#endif 187 188#if defined(AO_HAVE_XSIZE_compare_and_swap) \ 189 && !defined(AO_HAVE_XSIZE_fetch_and_add) 190 AO_INLINE XCTYPE 191 AO_XSIZE_fetch_and_add(volatile XCTYPE *addr, XCTYPE incr) 192 { 193 XCTYPE old; 194 195 do 196 { 197 old = *(XCTYPE *)addr; 198 } 199 while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap(addr, old, 200 old + incr))); 201 return old; 202 } 203# define AO_HAVE_XSIZE_fetch_and_add 204#endif 205 206#if defined(AO_HAVE_XSIZE_fetch_and_add_full) 207# if !defined(AO_HAVE_XSIZE_fetch_and_add_release) 208# define AO_XSIZE_fetch_and_add_release(addr, val) \ 209 AO_XSIZE_fetch_and_add_full(addr, val) 210# define AO_HAVE_XSIZE_fetch_and_add_release 211# endif 212# if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire) 213# define AO_XSIZE_fetch_and_add_acquire(addr, val) \ 214 AO_XSIZE_fetch_and_add_full(addr, val) 215# define AO_HAVE_XSIZE_fetch_and_add_acquire 216# endif 217# if !defined(AO_HAVE_XSIZE_fetch_and_add_write) 218# define AO_XSIZE_fetch_and_add_write(addr, val) \ 219 AO_XSIZE_fetch_and_add_full(addr, val) 220# define AO_HAVE_XSIZE_fetch_and_add_write 221# endif 222# if !defined(AO_HAVE_XSIZE_fetch_and_add_read) 223# define AO_XSIZE_fetch_and_add_read(addr, val) \ 224 AO_XSIZE_fetch_and_add_full(addr, val) 225# define AO_HAVE_XSIZE_fetch_and_add_read 226# endif 227#endif /* AO_HAVE_XSIZE_fetch_and_add_full */ 228 229#if defined(AO_HAVE_XSIZE_fetch_and_add) && defined(AO_HAVE_nop_full) \ 230 && !defined(AO_HAVE_XSIZE_fetch_and_add_acquire) 231 AO_INLINE XCTYPE 232 AO_XSIZE_fetch_and_add_acquire(volatile XCTYPE *addr, XCTYPE incr) 233 { 234 XCTYPE result = AO_XSIZE_fetch_and_add(addr, incr); 235 AO_nop_full(); 236 return result; 237 } 238# define AO_HAVE_XSIZE_fetch_and_add_acquire 239#endif 240#if defined(AO_HAVE_XSIZE_fetch_and_add) && defined(AO_HAVE_nop_full) \ 241 && !defined(AO_HAVE_XSIZE_fetch_and_add_release) 242# define AO_XSIZE_fetch_and_add_release(addr, incr) \ 243 (AO_nop_full(), AO_XSIZE_fetch_and_add(addr, incr)) 244# define AO_HAVE_XSIZE_fetch_and_add_release 245#endif 246 247#if !defined(AO_HAVE_XSIZE_fetch_and_add) \ 248 && defined(AO_HAVE_XSIZE_fetch_and_add_release) 249# define AO_XSIZE_fetch_and_add(addr, val) \ 250 AO_XSIZE_fetch_and_add_release(addr, val) 251# define AO_HAVE_XSIZE_fetch_and_add 252#endif 253#if !defined(AO_HAVE_XSIZE_fetch_and_add) \ 254 && defined(AO_HAVE_XSIZE_fetch_and_add_acquire) 255# define AO_XSIZE_fetch_and_add(addr, val) \ 256 AO_XSIZE_fetch_and_add_acquire(addr, val) 257# define AO_HAVE_XSIZE_fetch_and_add 258#endif 259#if !defined(AO_HAVE_XSIZE_fetch_and_add) \ 260 && defined(AO_HAVE_XSIZE_fetch_and_add_write) 261# define AO_XSIZE_fetch_and_add(addr, val) \ 262 AO_XSIZE_fetch_and_add_write(addr, val) 263# define AO_HAVE_XSIZE_fetch_and_add 264#endif 265#if !defined(AO_HAVE_XSIZE_fetch_and_add) \ 266 && defined(AO_HAVE_XSIZE_fetch_and_add_read) 267# define AO_XSIZE_fetch_and_add(addr, val) \ 268 AO_XSIZE_fetch_and_add_read(addr, val) 269# define AO_HAVE_XSIZE_fetch_and_add 270#endif 271 272#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \ 273 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_XSIZE_fetch_and_add_full) 274# define AO_XSIZE_fetch_and_add_full(addr, val) \ 275 (AO_nop_full(), AO_XSIZE_fetch_and_add_acquire(addr, val)) 276# define AO_HAVE_XSIZE_fetch_and_add_full 277#endif 278 279#if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ 280 && defined(AO_HAVE_XSIZE_fetch_and_add_write) 281# define AO_XSIZE_fetch_and_add_release_write(addr, val) \ 282 AO_XSIZE_fetch_and_add_write(addr, val) 283# define AO_HAVE_XSIZE_fetch_and_add_release_write 284#endif 285#if !defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ 286 && defined(AO_HAVE_XSIZE_fetch_and_add_release) 287# define AO_XSIZE_fetch_and_add_release_write(addr, val) \ 288 AO_XSIZE_fetch_and_add_release(addr, val) 289# define AO_HAVE_XSIZE_fetch_and_add_release_write 290#endif 291 292#if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ 293 && defined(AO_HAVE_XSIZE_fetch_and_add_read) 294# define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \ 295 AO_XSIZE_fetch_and_add_read(addr, val) 296# define AO_HAVE_XSIZE_fetch_and_add_acquire_read 297#endif 298#if !defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ 299 && defined(AO_HAVE_XSIZE_fetch_and_add_acquire) 300# define AO_XSIZE_fetch_and_add_acquire_read(addr, val) \ 301 AO_XSIZE_fetch_and_add_acquire(addr, val) 302# define AO_HAVE_XSIZE_fetch_and_add_acquire_read 303#endif 304 305#ifdef AO_NO_DD_ORDERING 306# if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) 307# define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \ 308 AO_XSIZE_fetch_and_add_acquire_read(addr, val) 309# define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read 310# endif 311#else 312# if defined(AO_HAVE_XSIZE_fetch_and_add) 313# define AO_XSIZE_fetch_and_add_dd_acquire_read(addr, val) \ 314 AO_XSIZE_fetch_and_add(addr, val) 315# define AO_HAVE_XSIZE_fetch_and_add_dd_acquire_read 316# endif 317#endif /* !AO_NO_DD_ORDERING */ 318 319/* XSIZE_fetch_and_add1 */ 320#if defined(AO_HAVE_XSIZE_fetch_and_add_full) \ 321 && !defined(AO_HAVE_XSIZE_fetch_and_add1_full) 322# define AO_XSIZE_fetch_and_add1_full(addr) \ 323 AO_XSIZE_fetch_and_add_full(addr, 1) 324# define AO_HAVE_XSIZE_fetch_and_add1_full 325#endif 326#if defined(AO_HAVE_XSIZE_fetch_and_add_release) \ 327 && !defined(AO_HAVE_XSIZE_fetch_and_add1_release) 328# define AO_XSIZE_fetch_and_add1_release(addr) \ 329 AO_XSIZE_fetch_and_add_release(addr, 1) 330# define AO_HAVE_XSIZE_fetch_and_add1_release 331#endif 332#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \ 333 && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) 334# define AO_XSIZE_fetch_and_add1_acquire(addr) \ 335 AO_XSIZE_fetch_and_add_acquire(addr, 1) 336# define AO_HAVE_XSIZE_fetch_and_add1_acquire 337#endif 338#if defined(AO_HAVE_XSIZE_fetch_and_add_write) \ 339 && !defined(AO_HAVE_XSIZE_fetch_and_add1_write) 340# define AO_XSIZE_fetch_and_add1_write(addr) \ 341 AO_XSIZE_fetch_and_add_write(addr, 1) 342# define AO_HAVE_XSIZE_fetch_and_add1_write 343#endif 344#if defined(AO_HAVE_XSIZE_fetch_and_add_read) \ 345 && !defined(AO_HAVE_XSIZE_fetch_and_add1_read) 346# define AO_XSIZE_fetch_and_add1_read(addr) \ 347 AO_XSIZE_fetch_and_add_read(addr, 1) 348# define AO_HAVE_XSIZE_fetch_and_add1_read 349#endif 350#if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ 351 && !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) 352# define AO_XSIZE_fetch_and_add1_release_write(addr) \ 353 AO_XSIZE_fetch_and_add_release_write(addr, 1) 354# define AO_HAVE_XSIZE_fetch_and_add1_release_write 355#endif 356#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ 357 && !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) 358# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \ 359 AO_XSIZE_fetch_and_add_acquire_read(addr, 1) 360# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read 361#endif 362#if defined(AO_HAVE_XSIZE_fetch_and_add) \ 363 && !defined(AO_HAVE_XSIZE_fetch_and_add1) 364# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add(addr, 1) 365# define AO_HAVE_XSIZE_fetch_and_add1 366#endif 367 368#if defined(AO_HAVE_XSIZE_fetch_and_add1_full) 369# if !defined(AO_HAVE_XSIZE_fetch_and_add1_release) 370# define AO_XSIZE_fetch_and_add1_release(addr) \ 371 AO_XSIZE_fetch_and_add1_full(addr) 372# define AO_HAVE_XSIZE_fetch_and_add1_release 373# endif 374# if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) 375# define AO_XSIZE_fetch_and_add1_acquire(addr) \ 376 AO_XSIZE_fetch_and_add1_full(addr) 377# define AO_HAVE_XSIZE_fetch_and_add1_acquire 378# endif 379# if !defined(AO_HAVE_XSIZE_fetch_and_add1_write) 380# define AO_XSIZE_fetch_and_add1_write(addr) \ 381 AO_XSIZE_fetch_and_add1_full(addr) 382# define AO_HAVE_XSIZE_fetch_and_add1_write 383# endif 384# if !defined(AO_HAVE_XSIZE_fetch_and_add1_read) 385# define AO_XSIZE_fetch_and_add1_read(addr) \ 386 AO_XSIZE_fetch_and_add1_full(addr) 387# define AO_HAVE_XSIZE_fetch_and_add1_read 388# endif 389#endif /* AO_HAVE_XSIZE_fetch_and_add1_full */ 390 391#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ 392 && defined(AO_HAVE_XSIZE_fetch_and_add1_release) 393# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_release(addr) 394# define AO_HAVE_XSIZE_fetch_and_add1 395#endif 396#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ 397 && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) 398# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_acquire(addr) 399# define AO_HAVE_XSIZE_fetch_and_add1 400#endif 401#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ 402 && defined(AO_HAVE_XSIZE_fetch_and_add1_write) 403# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_write(addr) 404# define AO_HAVE_XSIZE_fetch_and_add1 405#endif 406#if !defined(AO_HAVE_XSIZE_fetch_and_add1) \ 407 && defined(AO_HAVE_XSIZE_fetch_and_add1_read) 408# define AO_XSIZE_fetch_and_add1(addr) AO_XSIZE_fetch_and_add1_read(addr) 409# define AO_HAVE_XSIZE_fetch_and_add1 410#endif 411 412#if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) \ 413 && defined(AO_HAVE_nop_full) \ 414 && !defined(AO_HAVE_XSIZE_fetch_and_add1_full) 415# define AO_XSIZE_fetch_and_add1_full(addr) \ 416 (AO_nop_full(), AO_XSIZE_fetch_and_add1_acquire(addr)) 417# define AO_HAVE_XSIZE_fetch_and_add1_full 418#endif 419 420#if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \ 421 && defined(AO_HAVE_XSIZE_fetch_and_add1_write) 422# define AO_XSIZE_fetch_and_add1_release_write(addr) \ 423 AO_XSIZE_fetch_and_add1_write(addr) 424# define AO_HAVE_XSIZE_fetch_and_add1_release_write 425#endif 426#if !defined(AO_HAVE_XSIZE_fetch_and_add1_release_write) \ 427 && defined(AO_HAVE_XSIZE_fetch_and_add1_release) 428# define AO_XSIZE_fetch_and_add1_release_write(addr) \ 429 AO_XSIZE_fetch_and_add1_release(addr) 430# define AO_HAVE_XSIZE_fetch_and_add1_release_write 431#endif 432#if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \ 433 && defined(AO_HAVE_XSIZE_fetch_and_add1_read) 434# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \ 435 AO_XSIZE_fetch_and_add1_read(addr) 436# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read 437#endif 438#if !defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) \ 439 && defined(AO_HAVE_XSIZE_fetch_and_add1_acquire) 440# define AO_XSIZE_fetch_and_add1_acquire_read(addr) \ 441 AO_XSIZE_fetch_and_add1_acquire(addr) 442# define AO_HAVE_XSIZE_fetch_and_add1_acquire_read 443#endif 444 445#ifdef AO_NO_DD_ORDERING 446# if defined(AO_HAVE_XSIZE_fetch_and_add1_acquire_read) 447# define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \ 448 AO_XSIZE_fetch_and_add1_acquire_read(addr) 449# define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read 450# endif 451#else 452# if defined(AO_HAVE_XSIZE_fetch_and_add1) 453# define AO_XSIZE_fetch_and_add1_dd_acquire_read(addr) \ 454 AO_XSIZE_fetch_and_add1(addr) 455# define AO_HAVE_XSIZE_fetch_and_add1_dd_acquire_read 456# endif 457#endif /* !AO_NO_DD_ORDERING */ 458 459/* XSIZE_fetch_and_sub1 */ 460#if defined(AO_HAVE_XSIZE_fetch_and_add_full) \ 461 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full) 462# define AO_XSIZE_fetch_and_sub1_full(addr) \ 463 AO_XSIZE_fetch_and_add_full(addr, (XCTYPE)(-1)) 464# define AO_HAVE_XSIZE_fetch_and_sub1_full 465#endif 466#if defined(AO_HAVE_XSIZE_fetch_and_add_release) \ 467 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release) 468# define AO_XSIZE_fetch_and_sub1_release(addr) \ 469 AO_XSIZE_fetch_and_add_release(addr, (XCTYPE)(-1)) 470# define AO_HAVE_XSIZE_fetch_and_sub1_release 471#endif 472#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire) \ 473 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) 474# define AO_XSIZE_fetch_and_sub1_acquire(addr) \ 475 AO_XSIZE_fetch_and_add_acquire(addr, (XCTYPE)(-1)) 476# define AO_HAVE_XSIZE_fetch_and_sub1_acquire 477#endif 478#if defined(AO_HAVE_XSIZE_fetch_and_add_write) \ 479 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_write) 480# define AO_XSIZE_fetch_and_sub1_write(addr) \ 481 AO_XSIZE_fetch_and_add_write(addr, (XCTYPE)(-1)) 482# define AO_HAVE_XSIZE_fetch_and_sub1_write 483#endif 484#if defined(AO_HAVE_XSIZE_fetch_and_add_read) \ 485 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_read) 486# define AO_XSIZE_fetch_and_sub1_read(addr) \ 487 AO_XSIZE_fetch_and_add_read(addr, (XCTYPE)(-1)) 488# define AO_HAVE_XSIZE_fetch_and_sub1_read 489#endif 490#if defined(AO_HAVE_XSIZE_fetch_and_add_release_write) \ 491 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) 492# define AO_XSIZE_fetch_and_sub1_release_write(addr) \ 493 AO_XSIZE_fetch_and_add_release_write(addr, (XCTYPE)(-1)) 494# define AO_HAVE_XSIZE_fetch_and_sub1_release_write 495#endif 496#if defined(AO_HAVE_XSIZE_fetch_and_add_acquire_read) \ 497 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) 498# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \ 499 AO_XSIZE_fetch_and_add_acquire_read(addr, (XCTYPE)(-1)) 500# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read 501#endif 502#if defined(AO_HAVE_XSIZE_fetch_and_add) \ 503 && !defined(AO_HAVE_XSIZE_fetch_and_sub1) 504# define AO_XSIZE_fetch_and_sub1(addr) \ 505 AO_XSIZE_fetch_and_add(addr, (XCTYPE)(-1)) 506# define AO_HAVE_XSIZE_fetch_and_sub1 507#endif 508 509#if defined(AO_HAVE_XSIZE_fetch_and_sub1_full) 510# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release) 511# define AO_XSIZE_fetch_and_sub1_release(addr) \ 512 AO_XSIZE_fetch_and_sub1_full(addr) 513# define AO_HAVE_XSIZE_fetch_and_sub1_release 514# endif 515# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) 516# define AO_XSIZE_fetch_and_sub1_acquire(addr) \ 517 AO_XSIZE_fetch_and_sub1_full(addr) 518# define AO_HAVE_XSIZE_fetch_and_sub1_acquire 519# endif 520# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_write) 521# define AO_XSIZE_fetch_and_sub1_write(addr) \ 522 AO_XSIZE_fetch_and_sub1_full(addr) 523# define AO_HAVE_XSIZE_fetch_and_sub1_write 524# endif 525# if !defined(AO_HAVE_XSIZE_fetch_and_sub1_read) 526# define AO_XSIZE_fetch_and_sub1_read(addr) \ 527 AO_XSIZE_fetch_and_sub1_full(addr) 528# define AO_HAVE_XSIZE_fetch_and_sub1_read 529# endif 530#endif /* AO_HAVE_XSIZE_fetch_and_sub1_full */ 531 532#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ 533 && defined(AO_HAVE_XSIZE_fetch_and_sub1_release) 534# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_release(addr) 535# define AO_HAVE_XSIZE_fetch_and_sub1 536#endif 537#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ 538 && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) 539# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_acquire(addr) 540# define AO_HAVE_XSIZE_fetch_and_sub1 541#endif 542#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ 543 && defined(AO_HAVE_XSIZE_fetch_and_sub1_write) 544# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_write(addr) 545# define AO_HAVE_XSIZE_fetch_and_sub1 546#endif 547#if !defined(AO_HAVE_XSIZE_fetch_and_sub1) \ 548 && defined(AO_HAVE_XSIZE_fetch_and_sub1_read) 549# define AO_XSIZE_fetch_and_sub1(addr) AO_XSIZE_fetch_and_sub1_read(addr) 550# define AO_HAVE_XSIZE_fetch_and_sub1 551#endif 552 553#if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) \ 554 && defined(AO_HAVE_nop_full) \ 555 && !defined(AO_HAVE_XSIZE_fetch_and_sub1_full) 556# define AO_XSIZE_fetch_and_sub1_full(addr) \ 557 (AO_nop_full(), AO_XSIZE_fetch_and_sub1_acquire(addr)) 558# define AO_HAVE_XSIZE_fetch_and_sub1_full 559#endif 560 561#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \ 562 && defined(AO_HAVE_XSIZE_fetch_and_sub1_write) 563# define AO_XSIZE_fetch_and_sub1_release_write(addr) \ 564 AO_XSIZE_fetch_and_sub1_write(addr) 565# define AO_HAVE_XSIZE_fetch_and_sub1_release_write 566#endif 567#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_release_write) \ 568 && defined(AO_HAVE_XSIZE_fetch_and_sub1_release) 569# define AO_XSIZE_fetch_and_sub1_release_write(addr) \ 570 AO_XSIZE_fetch_and_sub1_release(addr) 571# define AO_HAVE_XSIZE_fetch_and_sub1_release_write 572#endif 573#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \ 574 && defined(AO_HAVE_XSIZE_fetch_and_sub1_read) 575# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \ 576 AO_XSIZE_fetch_and_sub1_read(addr) 577# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read 578#endif 579#if !defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) \ 580 && defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire) 581# define AO_XSIZE_fetch_and_sub1_acquire_read(addr) \ 582 AO_XSIZE_fetch_and_sub1_acquire(addr) 583# define AO_HAVE_XSIZE_fetch_and_sub1_acquire_read 584#endif 585 586#ifdef AO_NO_DD_ORDERING 587# if defined(AO_HAVE_XSIZE_fetch_and_sub1_acquire_read) 588# define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \ 589 AO_XSIZE_fetch_and_sub1_acquire_read(addr) 590# define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read 591# endif 592#else 593# if defined(AO_HAVE_XSIZE_fetch_and_sub1) 594# define AO_XSIZE_fetch_and_sub1_dd_acquire_read(addr) \ 595 AO_XSIZE_fetch_and_sub1(addr) 596# define AO_HAVE_XSIZE_fetch_and_sub1_dd_acquire_read 597# endif 598#endif /* !AO_NO_DD_ORDERING */ 599 600/* XSIZE_and */ 601#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ 602 && !defined(AO_HAVE_XSIZE_and_full) 603 AO_INLINE void 604 AO_XSIZE_and_full(volatile XCTYPE *addr, XCTYPE value) 605 { 606 XCTYPE old; 607 608 do 609 { 610 old = *(XCTYPE *)addr; 611 } 612 while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, 613 old & value))); 614 } 615# define AO_HAVE_XSIZE_and_full 616#endif 617 618#if defined(AO_HAVE_XSIZE_and_full) 619# if !defined(AO_HAVE_XSIZE_and_release) 620# define AO_XSIZE_and_release(addr, val) AO_XSIZE_and_full(addr, val) 621# define AO_HAVE_XSIZE_and_release 622# endif 623# if !defined(AO_HAVE_XSIZE_and_acquire) 624# define AO_XSIZE_and_acquire(addr, val) AO_XSIZE_and_full(addr, val) 625# define AO_HAVE_XSIZE_and_acquire 626# endif 627# if !defined(AO_HAVE_XSIZE_and_write) 628# define AO_XSIZE_and_write(addr, val) AO_XSIZE_and_full(addr, val) 629# define AO_HAVE_XSIZE_and_write 630# endif 631# if !defined(AO_HAVE_XSIZE_and_read) 632# define AO_XSIZE_and_read(addr, val) AO_XSIZE_and_full(addr, val) 633# define AO_HAVE_XSIZE_and_read 634# endif 635#endif /* AO_HAVE_XSIZE_and_full */ 636 637#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_release) 638# define AO_XSIZE_and(addr, val) AO_XSIZE_and_release(addr, val) 639# define AO_HAVE_XSIZE_and 640#endif 641#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_acquire) 642# define AO_XSIZE_and(addr, val) AO_XSIZE_and_acquire(addr, val) 643# define AO_HAVE_XSIZE_and 644#endif 645#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_write) 646# define AO_XSIZE_and(addr, val) AO_XSIZE_and_write(addr, val) 647# define AO_HAVE_XSIZE_and 648#endif 649#if !defined(AO_HAVE_XSIZE_and) && defined(AO_HAVE_XSIZE_and_read) 650# define AO_XSIZE_and(addr, val) AO_XSIZE_and_read(addr, val) 651# define AO_HAVE_XSIZE_and 652#endif 653 654#if defined(AO_HAVE_XSIZE_and_acquire) && defined(AO_HAVE_nop_full) \ 655 && !defined(AO_HAVE_XSIZE_and_full) 656# define AO_XSIZE_and_full(addr, val) \ 657 (AO_nop_full(), AO_XSIZE_and_acquire(addr, val)) 658# define AO_HAVE_XSIZE_and_full 659#endif 660 661#if !defined(AO_HAVE_XSIZE_and_release_write) \ 662 && defined(AO_HAVE_XSIZE_and_write) 663# define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_write(addr, val) 664# define AO_HAVE_XSIZE_and_release_write 665#endif 666#if !defined(AO_HAVE_XSIZE_and_release_write) \ 667 && defined(AO_HAVE_XSIZE_and_release) 668# define AO_XSIZE_and_release_write(addr, val) AO_XSIZE_and_release(addr, val) 669# define AO_HAVE_XSIZE_and_release_write 670#endif 671#if !defined(AO_HAVE_XSIZE_and_acquire_read) \ 672 && defined(AO_HAVE_XSIZE_and_read) 673# define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_read(addr, val) 674# define AO_HAVE_XSIZE_and_acquire_read 675#endif 676#if !defined(AO_HAVE_XSIZE_and_acquire_read) \ 677 && defined(AO_HAVE_XSIZE_and_acquire) 678# define AO_XSIZE_and_acquire_read(addr, val) AO_XSIZE_and_acquire(addr, val) 679# define AO_HAVE_XSIZE_and_acquire_read 680#endif 681 682/* XSIZE_or */ 683#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ 684 && !defined(AO_HAVE_XSIZE_or_full) 685 AO_INLINE void 686 AO_XSIZE_or_full(volatile XCTYPE *addr, XCTYPE value) 687 { 688 XCTYPE old; 689 690 do 691 { 692 old = *(XCTYPE *)addr; 693 } 694 while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, 695 old | value))); 696 } 697# define AO_HAVE_XSIZE_or_full 698#endif 699 700#if defined(AO_HAVE_XSIZE_or_full) 701# if !defined(AO_HAVE_XSIZE_or_release) 702# define AO_XSIZE_or_release(addr, val) AO_XSIZE_or_full(addr, val) 703# define AO_HAVE_XSIZE_or_release 704# endif 705# if !defined(AO_HAVE_XSIZE_or_acquire) 706# define AO_XSIZE_or_acquire(addr, val) AO_XSIZE_or_full(addr, val) 707# define AO_HAVE_XSIZE_or_acquire 708# endif 709# if !defined(AO_HAVE_XSIZE_or_write) 710# define AO_XSIZE_or_write(addr, val) AO_XSIZE_or_full(addr, val) 711# define AO_HAVE_XSIZE_or_write 712# endif 713# if !defined(AO_HAVE_XSIZE_or_read) 714# define AO_XSIZE_or_read(addr, val) AO_XSIZE_or_full(addr, val) 715# define AO_HAVE_XSIZE_or_read 716# endif 717#endif /* AO_HAVE_XSIZE_or_full */ 718 719#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_release) 720# define AO_XSIZE_or(addr, val) AO_XSIZE_or_release(addr, val) 721# define AO_HAVE_XSIZE_or 722#endif 723#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_acquire) 724# define AO_XSIZE_or(addr, val) AO_XSIZE_or_acquire(addr, val) 725# define AO_HAVE_XSIZE_or 726#endif 727#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_write) 728# define AO_XSIZE_or(addr, val) AO_XSIZE_or_write(addr, val) 729# define AO_HAVE_XSIZE_or 730#endif 731#if !defined(AO_HAVE_XSIZE_or) && defined(AO_HAVE_XSIZE_or_read) 732# define AO_XSIZE_or(addr, val) AO_XSIZE_or_read(addr, val) 733# define AO_HAVE_XSIZE_or 734#endif 735 736#if defined(AO_HAVE_XSIZE_or_acquire) && defined(AO_HAVE_nop_full) \ 737 && !defined(AO_HAVE_XSIZE_or_full) 738# define AO_XSIZE_or_full(addr, val) \ 739 (AO_nop_full(), AO_XSIZE_or_acquire(addr, val)) 740# define AO_HAVE_XSIZE_or_full 741#endif 742 743#if !defined(AO_HAVE_XSIZE_or_release_write) \ 744 && defined(AO_HAVE_XSIZE_or_write) 745# define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_write(addr, val) 746# define AO_HAVE_XSIZE_or_release_write 747#endif 748#if !defined(AO_HAVE_XSIZE_or_release_write) \ 749 && defined(AO_HAVE_XSIZE_or_release) 750# define AO_XSIZE_or_release_write(addr, val) AO_XSIZE_or_release(addr, val) 751# define AO_HAVE_XSIZE_or_release_write 752#endif 753#if !defined(AO_HAVE_XSIZE_or_acquire_read) && defined(AO_HAVE_XSIZE_or_read) 754# define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_read(addr, val) 755# define AO_HAVE_XSIZE_or_acquire_read 756#endif 757#if !defined(AO_HAVE_XSIZE_or_acquire_read) \ 758 && defined(AO_HAVE_XSIZE_or_acquire) 759# define AO_XSIZE_or_acquire_read(addr, val) AO_XSIZE_or_acquire(addr, val) 760# define AO_HAVE_XSIZE_or_acquire_read 761#endif 762 763/* XSIZE_xor */ 764#if defined(AO_HAVE_XSIZE_compare_and_swap_full) \ 765 && !defined(AO_HAVE_XSIZE_xor_full) 766 AO_INLINE void 767 AO_XSIZE_xor_full(volatile XCTYPE *addr, XCTYPE value) 768 { 769 XCTYPE old; 770 771 do 772 { 773 old = *(XCTYPE *)addr; 774 } 775 while (AO_EXPECT_FALSE(!AO_XSIZE_compare_and_swap_full(addr, old, 776 old ^ value))); 777 } 778# define AO_HAVE_XSIZE_xor_full 779#endif 780 781#if defined(AO_HAVE_XSIZE_xor_full) 782# if !defined(AO_HAVE_XSIZE_xor_release) 783# define AO_XSIZE_xor_release(addr, val) AO_XSIZE_xor_full(addr, val) 784# define AO_HAVE_XSIZE_xor_release 785# endif 786# if !defined(AO_HAVE_XSIZE_xor_acquire) 787# define AO_XSIZE_xor_acquire(addr, val) AO_XSIZE_xor_full(addr, val) 788# define AO_HAVE_XSIZE_xor_acquire 789# endif 790# if !defined(AO_HAVE_XSIZE_xor_write) 791# define AO_XSIZE_xor_write(addr, val) AO_XSIZE_xor_full(addr, val) 792# define AO_HAVE_XSIZE_xor_write 793# endif 794# if !defined(AO_HAVE_XSIZE_xor_read) 795# define AO_XSIZE_xor_read(addr, val) AO_XSIZE_xor_full(addr, val) 796# define AO_HAVE_XSIZE_xor_read 797# endif 798#endif /* AO_HAVE_XSIZE_xor_full */ 799 800#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_release) 801# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_release(addr, val) 802# define AO_HAVE_XSIZE_xor 803#endif 804#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_acquire) 805# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_acquire(addr, val) 806# define AO_HAVE_XSIZE_xor 807#endif 808#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_write) 809# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_write(addr, val) 810# define AO_HAVE_XSIZE_xor 811#endif 812#if !defined(AO_HAVE_XSIZE_xor) && defined(AO_HAVE_XSIZE_xor_read) 813# define AO_XSIZE_xor(addr, val) AO_XSIZE_xor_read(addr, val) 814# define AO_HAVE_XSIZE_xor 815#endif 816 817#if defined(AO_HAVE_XSIZE_xor_acquire) && defined(AO_HAVE_nop_full) \ 818 && !defined(AO_HAVE_XSIZE_xor_full) 819# define AO_XSIZE_xor_full(addr, val) \ 820 (AO_nop_full(), AO_XSIZE_xor_acquire(addr, val)) 821# define AO_HAVE_XSIZE_xor_full 822#endif 823 824#if !defined(AO_HAVE_XSIZE_xor_release_write) \ 825 && defined(AO_HAVE_XSIZE_xor_write) 826# define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_write(addr, val) 827# define AO_HAVE_XSIZE_xor_release_write 828#endif 829#if !defined(AO_HAVE_XSIZE_xor_release_write) \ 830 && defined(AO_HAVE_XSIZE_xor_release) 831# define AO_XSIZE_xor_release_write(addr, val) AO_XSIZE_xor_release(addr, val) 832# define AO_HAVE_XSIZE_xor_release_write 833#endif 834#if !defined(AO_HAVE_XSIZE_xor_acquire_read) \ 835 && defined(AO_HAVE_XSIZE_xor_read) 836# define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_read(addr, val) 837# define AO_HAVE_XSIZE_xor_acquire_read 838#endif 839#if !defined(AO_HAVE_XSIZE_xor_acquire_read) \ 840 && defined(AO_HAVE_XSIZE_xor_acquire) 841# define AO_XSIZE_xor_acquire_read(addr, val) AO_XSIZE_xor_acquire(addr, val) 842# define AO_HAVE_XSIZE_xor_acquire_read 843#endif 844 845/* XSIZE_and/or/xor_dd_acquire_read are meaningless. */ 846