1 /* 2 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a copy 5 * of this software and associated documentation files (the "Software"), to deal 6 * in the Software without restriction, including without limitation the rights 7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 * copies of the Software, and to permit persons to whom the Software is 9 * furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 * SOFTWARE. 21 */ 22 23 /* char_compare_and_swap (based on fetch_compare_and_swap) */ 24 #if defined(AO_HAVE_char_fetch_compare_and_swap_full) \ 25 && !defined(AO_HAVE_char_compare_and_swap_full) 26 AO_INLINE int AO_char_compare_and_swap_full(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)27 AO_char_compare_and_swap_full(volatile unsigned/**/char *addr, unsigned/**/char old_val, 28 unsigned/**/char new_val) 29 { 30 return AO_char_fetch_compare_and_swap_full(addr, old_val, new_val) 31 == old_val; 32 } 33 # define AO_HAVE_char_compare_and_swap_full 34 #endif 35 36 #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire) \ 37 && !defined(AO_HAVE_char_compare_and_swap_acquire) 38 AO_INLINE int AO_char_compare_and_swap_acquire(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)39 AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr, unsigned/**/char old_val, 40 unsigned/**/char new_val) 41 { 42 return AO_char_fetch_compare_and_swap_acquire(addr, old_val, new_val) 43 == old_val; 44 } 45 # define AO_HAVE_char_compare_and_swap_acquire 46 #endif 47 48 #if defined(AO_HAVE_char_fetch_compare_and_swap_release) \ 49 && !defined(AO_HAVE_char_compare_and_swap_release) 50 AO_INLINE int AO_char_compare_and_swap_release(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)51 AO_char_compare_and_swap_release(volatile unsigned/**/char *addr, unsigned/**/char old_val, 52 unsigned/**/char new_val) 53 { 54 return AO_char_fetch_compare_and_swap_release(addr, old_val, new_val) 55 == old_val; 56 } 57 # define AO_HAVE_char_compare_and_swap_release 58 #endif 59 60 #if defined(AO_HAVE_char_fetch_compare_and_swap_write) \ 61 && !defined(AO_HAVE_char_compare_and_swap_write) 62 AO_INLINE int AO_char_compare_and_swap_write(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)63 AO_char_compare_and_swap_write(volatile unsigned/**/char *addr, unsigned/**/char old_val, 64 unsigned/**/char new_val) 65 { 66 return AO_char_fetch_compare_and_swap_write(addr, old_val, new_val) 67 == old_val; 68 } 69 # define AO_HAVE_char_compare_and_swap_write 70 #endif 71 72 #if defined(AO_HAVE_char_fetch_compare_and_swap_read) \ 73 && !defined(AO_HAVE_char_compare_and_swap_read) 74 AO_INLINE int AO_char_compare_and_swap_read(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)75 AO_char_compare_and_swap_read(volatile unsigned/**/char *addr, unsigned/**/char old_val, 76 unsigned/**/char new_val) 77 { 78 return AO_char_fetch_compare_and_swap_read(addr, old_val, new_val) 79 == old_val; 80 } 81 # define AO_HAVE_char_compare_and_swap_read 82 #endif 83 84 #if defined(AO_HAVE_char_fetch_compare_and_swap) \ 85 && !defined(AO_HAVE_char_compare_and_swap) 86 AO_INLINE int AO_char_compare_and_swap(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)87 AO_char_compare_and_swap(volatile unsigned/**/char *addr, unsigned/**/char old_val, 88 unsigned/**/char new_val) 89 { 90 return AO_char_fetch_compare_and_swap(addr, old_val, new_val) == old_val; 91 } 92 # define AO_HAVE_char_compare_and_swap 93 #endif 94 95 #if defined(AO_HAVE_char_fetch_compare_and_swap_release_write) \ 96 && !defined(AO_HAVE_char_compare_and_swap_release_write) 97 AO_INLINE int AO_char_compare_and_swap_release_write(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)98 AO_char_compare_and_swap_release_write(volatile unsigned/**/char *addr, 99 unsigned/**/char old_val, unsigned/**/char new_val) 100 { 101 return AO_char_fetch_compare_and_swap_release_write(addr, old_val, 102 new_val) == old_val; 103 } 104 # define AO_HAVE_char_compare_and_swap_release_write 105 #endif 106 107 #if defined(AO_HAVE_char_fetch_compare_and_swap_acquire_read) \ 108 && !defined(AO_HAVE_char_compare_and_swap_acquire_read) 109 AO_INLINE int AO_char_compare_and_swap_acquire_read(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)110 AO_char_compare_and_swap_acquire_read(volatile unsigned/**/char *addr, 111 unsigned/**/char old_val, unsigned/**/char new_val) 112 { 113 return AO_char_fetch_compare_and_swap_acquire_read(addr, old_val, 114 new_val) == old_val; 115 } 116 # define AO_HAVE_char_compare_and_swap_acquire_read 117 #endif 118 119 #if defined(AO_HAVE_char_fetch_compare_and_swap_dd_acquire_read) \ 120 && !defined(AO_HAVE_char_compare_and_swap_dd_acquire_read) 121 AO_INLINE int AO_char_compare_and_swap_dd_acquire_read(volatile unsigned char * addr,unsigned char old_val,unsigned char new_val)122 AO_char_compare_and_swap_dd_acquire_read(volatile unsigned/**/char *addr, 123 unsigned/**/char old_val, unsigned/**/char new_val) 124 { 125 return AO_char_fetch_compare_and_swap_dd_acquire_read(addr, old_val, 126 new_val) == old_val; 127 } 128 # define AO_HAVE_char_compare_and_swap_dd_acquire_read 129 #endif 130 131 /* char_fetch_and_add */ 132 /* We first try to implement fetch_and_add variants in terms of the */ 133 /* corresponding compare_and_swap variants to minimize adding barriers. */ 134 #if defined(AO_HAVE_char_compare_and_swap_full) \ 135 && !defined(AO_HAVE_char_fetch_and_add_full) 136 AO_INLINE unsigned/**/char AO_char_fetch_and_add_full(volatile unsigned char * addr,unsigned char incr)137 AO_char_fetch_and_add_full(volatile unsigned/**/char *addr, unsigned/**/char incr) 138 { 139 unsigned/**/char old; 140 141 do 142 { 143 old = *(unsigned/**/char *)addr; 144 } 145 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, 146 old + incr))); 147 return old; 148 } 149 # define AO_HAVE_char_fetch_and_add_full 150 #endif 151 152 #if defined(AO_HAVE_char_compare_and_swap_acquire) \ 153 && !defined(AO_HAVE_char_fetch_and_add_acquire) 154 AO_INLINE unsigned/**/char AO_char_fetch_and_add_acquire(volatile unsigned char * addr,unsigned char incr)155 AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr) 156 { 157 unsigned/**/char old; 158 159 do 160 { 161 old = *(unsigned/**/char *)addr; 162 } 163 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_acquire(addr, old, 164 old + incr))); 165 return old; 166 } 167 # define AO_HAVE_char_fetch_and_add_acquire 168 #endif 169 170 #if defined(AO_HAVE_char_compare_and_swap_release) \ 171 && !defined(AO_HAVE_char_fetch_and_add_release) 172 AO_INLINE unsigned/**/char AO_char_fetch_and_add_release(volatile unsigned char * addr,unsigned char incr)173 AO_char_fetch_and_add_release(volatile unsigned/**/char *addr, unsigned/**/char incr) 174 { 175 unsigned/**/char old; 176 177 do 178 { 179 old = *(unsigned/**/char *)addr; 180 } 181 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_release(addr, old, 182 old + incr))); 183 return old; 184 } 185 # define AO_HAVE_char_fetch_and_add_release 186 #endif 187 188 #if defined(AO_HAVE_char_compare_and_swap) \ 189 && !defined(AO_HAVE_char_fetch_and_add) 190 AO_INLINE unsigned/**/char AO_char_fetch_and_add(volatile unsigned char * addr,unsigned char incr)191 AO_char_fetch_and_add(volatile unsigned/**/char *addr, unsigned/**/char incr) 192 { 193 unsigned/**/char old; 194 195 do 196 { 197 old = *(unsigned/**/char *)addr; 198 } 199 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap(addr, old, 200 old + incr))); 201 return old; 202 } 203 # define AO_HAVE_char_fetch_and_add 204 #endif 205 206 #if defined(AO_HAVE_char_fetch_and_add_full) 207 # if !defined(AO_HAVE_char_fetch_and_add_release) 208 # define AO_char_fetch_and_add_release(addr, val) \ 209 AO_char_fetch_and_add_full(addr, val) 210 # define AO_HAVE_char_fetch_and_add_release 211 # endif 212 # if !defined(AO_HAVE_char_fetch_and_add_acquire) 213 # define AO_char_fetch_and_add_acquire(addr, val) \ 214 AO_char_fetch_and_add_full(addr, val) 215 # define AO_HAVE_char_fetch_and_add_acquire 216 # endif 217 # if !defined(AO_HAVE_char_fetch_and_add_write) 218 # define AO_char_fetch_and_add_write(addr, val) \ 219 AO_char_fetch_and_add_full(addr, val) 220 # define AO_HAVE_char_fetch_and_add_write 221 # endif 222 # if !defined(AO_HAVE_char_fetch_and_add_read) 223 # define AO_char_fetch_and_add_read(addr, val) \ 224 AO_char_fetch_and_add_full(addr, val) 225 # define AO_HAVE_char_fetch_and_add_read 226 # endif 227 #endif /* AO_HAVE_char_fetch_and_add_full */ 228 229 #if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \ 230 && !defined(AO_HAVE_char_fetch_and_add_acquire) 231 AO_INLINE unsigned/**/char AO_char_fetch_and_add_acquire(volatile unsigned char * addr,unsigned char incr)232 AO_char_fetch_and_add_acquire(volatile unsigned/**/char *addr, unsigned/**/char incr) 233 { 234 unsigned/**/char result = AO_char_fetch_and_add(addr, incr); 235 AO_nop_full(); 236 return result; 237 } 238 # define AO_HAVE_char_fetch_and_add_acquire 239 #endif 240 #if defined(AO_HAVE_char_fetch_and_add) && defined(AO_HAVE_nop_full) \ 241 && !defined(AO_HAVE_char_fetch_and_add_release) 242 # define AO_char_fetch_and_add_release(addr, incr) \ 243 (AO_nop_full(), AO_char_fetch_and_add(addr, incr)) 244 # define AO_HAVE_char_fetch_and_add_release 245 #endif 246 247 #if !defined(AO_HAVE_char_fetch_and_add) \ 248 && defined(AO_HAVE_char_fetch_and_add_release) 249 # define AO_char_fetch_and_add(addr, val) \ 250 AO_char_fetch_and_add_release(addr, val) 251 # define AO_HAVE_char_fetch_and_add 252 #endif 253 #if !defined(AO_HAVE_char_fetch_and_add) \ 254 && defined(AO_HAVE_char_fetch_and_add_acquire) 255 # define AO_char_fetch_and_add(addr, val) \ 256 AO_char_fetch_and_add_acquire(addr, val) 257 # define AO_HAVE_char_fetch_and_add 258 #endif 259 #if !defined(AO_HAVE_char_fetch_and_add) \ 260 && defined(AO_HAVE_char_fetch_and_add_write) 261 # define AO_char_fetch_and_add(addr, val) \ 262 AO_char_fetch_and_add_write(addr, val) 263 # define AO_HAVE_char_fetch_and_add 264 #endif 265 #if !defined(AO_HAVE_char_fetch_and_add) \ 266 && defined(AO_HAVE_char_fetch_and_add_read) 267 # define AO_char_fetch_and_add(addr, val) \ 268 AO_char_fetch_and_add_read(addr, val) 269 # define AO_HAVE_char_fetch_and_add 270 #endif 271 272 #if defined(AO_HAVE_char_fetch_and_add_acquire) \ 273 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_char_fetch_and_add_full) 274 # define AO_char_fetch_and_add_full(addr, val) \ 275 (AO_nop_full(), AO_char_fetch_and_add_acquire(addr, val)) 276 # define AO_HAVE_char_fetch_and_add_full 277 #endif 278 279 #if !defined(AO_HAVE_char_fetch_and_add_release_write) \ 280 && defined(AO_HAVE_char_fetch_and_add_write) 281 # define AO_char_fetch_and_add_release_write(addr, val) \ 282 AO_char_fetch_and_add_write(addr, val) 283 # define AO_HAVE_char_fetch_and_add_release_write 284 #endif 285 #if !defined(AO_HAVE_char_fetch_and_add_release_write) \ 286 && defined(AO_HAVE_char_fetch_and_add_release) 287 # define AO_char_fetch_and_add_release_write(addr, val) \ 288 AO_char_fetch_and_add_release(addr, val) 289 # define AO_HAVE_char_fetch_and_add_release_write 290 #endif 291 292 #if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \ 293 && defined(AO_HAVE_char_fetch_and_add_read) 294 # define AO_char_fetch_and_add_acquire_read(addr, val) \ 295 AO_char_fetch_and_add_read(addr, val) 296 # define AO_HAVE_char_fetch_and_add_acquire_read 297 #endif 298 #if !defined(AO_HAVE_char_fetch_and_add_acquire_read) \ 299 && defined(AO_HAVE_char_fetch_and_add_acquire) 300 # define AO_char_fetch_and_add_acquire_read(addr, val) \ 301 AO_char_fetch_and_add_acquire(addr, val) 302 # define AO_HAVE_char_fetch_and_add_acquire_read 303 #endif 304 305 #ifdef AO_NO_DD_ORDERING 306 # if defined(AO_HAVE_char_fetch_and_add_acquire_read) 307 # define AO_char_fetch_and_add_dd_acquire_read(addr, val) \ 308 AO_char_fetch_and_add_acquire_read(addr, val) 309 # define AO_HAVE_char_fetch_and_add_dd_acquire_read 310 # endif 311 #else 312 # if defined(AO_HAVE_char_fetch_and_add) 313 # define AO_char_fetch_and_add_dd_acquire_read(addr, val) \ 314 AO_char_fetch_and_add(addr, val) 315 # define AO_HAVE_char_fetch_and_add_dd_acquire_read 316 # endif 317 #endif /* !AO_NO_DD_ORDERING */ 318 319 /* char_fetch_and_add1 */ 320 #if defined(AO_HAVE_char_fetch_and_add_full) \ 321 && !defined(AO_HAVE_char_fetch_and_add1_full) 322 # define AO_char_fetch_and_add1_full(addr) \ 323 AO_char_fetch_and_add_full(addr, 1) 324 # define AO_HAVE_char_fetch_and_add1_full 325 #endif 326 #if defined(AO_HAVE_char_fetch_and_add_release) \ 327 && !defined(AO_HAVE_char_fetch_and_add1_release) 328 # define AO_char_fetch_and_add1_release(addr) \ 329 AO_char_fetch_and_add_release(addr, 1) 330 # define AO_HAVE_char_fetch_and_add1_release 331 #endif 332 #if defined(AO_HAVE_char_fetch_and_add_acquire) \ 333 && !defined(AO_HAVE_char_fetch_and_add1_acquire) 334 # define AO_char_fetch_and_add1_acquire(addr) \ 335 AO_char_fetch_and_add_acquire(addr, 1) 336 # define AO_HAVE_char_fetch_and_add1_acquire 337 #endif 338 #if defined(AO_HAVE_char_fetch_and_add_write) \ 339 && !defined(AO_HAVE_char_fetch_and_add1_write) 340 # define AO_char_fetch_and_add1_write(addr) \ 341 AO_char_fetch_and_add_write(addr, 1) 342 # define AO_HAVE_char_fetch_and_add1_write 343 #endif 344 #if defined(AO_HAVE_char_fetch_and_add_read) \ 345 && !defined(AO_HAVE_char_fetch_and_add1_read) 346 # define AO_char_fetch_and_add1_read(addr) \ 347 AO_char_fetch_and_add_read(addr, 1) 348 # define AO_HAVE_char_fetch_and_add1_read 349 #endif 350 #if defined(AO_HAVE_char_fetch_and_add_release_write) \ 351 && !defined(AO_HAVE_char_fetch_and_add1_release_write) 352 # define AO_char_fetch_and_add1_release_write(addr) \ 353 AO_char_fetch_and_add_release_write(addr, 1) 354 # define AO_HAVE_char_fetch_and_add1_release_write 355 #endif 356 #if defined(AO_HAVE_char_fetch_and_add_acquire_read) \ 357 && !defined(AO_HAVE_char_fetch_and_add1_acquire_read) 358 # define AO_char_fetch_and_add1_acquire_read(addr) \ 359 AO_char_fetch_and_add_acquire_read(addr, 1) 360 # define AO_HAVE_char_fetch_and_add1_acquire_read 361 #endif 362 #if defined(AO_HAVE_char_fetch_and_add) \ 363 && !defined(AO_HAVE_char_fetch_and_add1) 364 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add(addr, 1) 365 # define AO_HAVE_char_fetch_and_add1 366 #endif 367 368 #if defined(AO_HAVE_char_fetch_and_add1_full) 369 # if !defined(AO_HAVE_char_fetch_and_add1_release) 370 # define AO_char_fetch_and_add1_release(addr) \ 371 AO_char_fetch_and_add1_full(addr) 372 # define AO_HAVE_char_fetch_and_add1_release 373 # endif 374 # if !defined(AO_HAVE_char_fetch_and_add1_acquire) 375 # define AO_char_fetch_and_add1_acquire(addr) \ 376 AO_char_fetch_and_add1_full(addr) 377 # define AO_HAVE_char_fetch_and_add1_acquire 378 # endif 379 # if !defined(AO_HAVE_char_fetch_and_add1_write) 380 # define AO_char_fetch_and_add1_write(addr) \ 381 AO_char_fetch_and_add1_full(addr) 382 # define AO_HAVE_char_fetch_and_add1_write 383 # endif 384 # if !defined(AO_HAVE_char_fetch_and_add1_read) 385 # define AO_char_fetch_and_add1_read(addr) \ 386 AO_char_fetch_and_add1_full(addr) 387 # define AO_HAVE_char_fetch_and_add1_read 388 # endif 389 #endif /* AO_HAVE_char_fetch_and_add1_full */ 390 391 #if !defined(AO_HAVE_char_fetch_and_add1) \ 392 && defined(AO_HAVE_char_fetch_and_add1_release) 393 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_release(addr) 394 # define AO_HAVE_char_fetch_and_add1 395 #endif 396 #if !defined(AO_HAVE_char_fetch_and_add1) \ 397 && defined(AO_HAVE_char_fetch_and_add1_acquire) 398 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_acquire(addr) 399 # define AO_HAVE_char_fetch_and_add1 400 #endif 401 #if !defined(AO_HAVE_char_fetch_and_add1) \ 402 && defined(AO_HAVE_char_fetch_and_add1_write) 403 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_write(addr) 404 # define AO_HAVE_char_fetch_and_add1 405 #endif 406 #if !defined(AO_HAVE_char_fetch_and_add1) \ 407 && defined(AO_HAVE_char_fetch_and_add1_read) 408 # define AO_char_fetch_and_add1(addr) AO_char_fetch_and_add1_read(addr) 409 # define AO_HAVE_char_fetch_and_add1 410 #endif 411 412 #if defined(AO_HAVE_char_fetch_and_add1_acquire) \ 413 && defined(AO_HAVE_nop_full) \ 414 && !defined(AO_HAVE_char_fetch_and_add1_full) 415 # define AO_char_fetch_and_add1_full(addr) \ 416 (AO_nop_full(), AO_char_fetch_and_add1_acquire(addr)) 417 # define AO_HAVE_char_fetch_and_add1_full 418 #endif 419 420 #if !defined(AO_HAVE_char_fetch_and_add1_release_write) \ 421 && defined(AO_HAVE_char_fetch_and_add1_write) 422 # define AO_char_fetch_and_add1_release_write(addr) \ 423 AO_char_fetch_and_add1_write(addr) 424 # define AO_HAVE_char_fetch_and_add1_release_write 425 #endif 426 #if !defined(AO_HAVE_char_fetch_and_add1_release_write) \ 427 && defined(AO_HAVE_char_fetch_and_add1_release) 428 # define AO_char_fetch_and_add1_release_write(addr) \ 429 AO_char_fetch_and_add1_release(addr) 430 # define AO_HAVE_char_fetch_and_add1_release_write 431 #endif 432 #if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \ 433 && defined(AO_HAVE_char_fetch_and_add1_read) 434 # define AO_char_fetch_and_add1_acquire_read(addr) \ 435 AO_char_fetch_and_add1_read(addr) 436 # define AO_HAVE_char_fetch_and_add1_acquire_read 437 #endif 438 #if !defined(AO_HAVE_char_fetch_and_add1_acquire_read) \ 439 && defined(AO_HAVE_char_fetch_and_add1_acquire) 440 # define AO_char_fetch_and_add1_acquire_read(addr) \ 441 AO_char_fetch_and_add1_acquire(addr) 442 # define AO_HAVE_char_fetch_and_add1_acquire_read 443 #endif 444 445 #ifdef AO_NO_DD_ORDERING 446 # if defined(AO_HAVE_char_fetch_and_add1_acquire_read) 447 # define AO_char_fetch_and_add1_dd_acquire_read(addr) \ 448 AO_char_fetch_and_add1_acquire_read(addr) 449 # define AO_HAVE_char_fetch_and_add1_dd_acquire_read 450 # endif 451 #else 452 # if defined(AO_HAVE_char_fetch_and_add1) 453 # define AO_char_fetch_and_add1_dd_acquire_read(addr) \ 454 AO_char_fetch_and_add1(addr) 455 # define AO_HAVE_char_fetch_and_add1_dd_acquire_read 456 # endif 457 #endif /* !AO_NO_DD_ORDERING */ 458 459 /* char_fetch_and_sub1 */ 460 #if defined(AO_HAVE_char_fetch_and_add_full) \ 461 && !defined(AO_HAVE_char_fetch_and_sub1_full) 462 # define AO_char_fetch_and_sub1_full(addr) \ 463 AO_char_fetch_and_add_full(addr, (unsigned/**/char)(-1)) 464 # define AO_HAVE_char_fetch_and_sub1_full 465 #endif 466 #if defined(AO_HAVE_char_fetch_and_add_release) \ 467 && !defined(AO_HAVE_char_fetch_and_sub1_release) 468 # define AO_char_fetch_and_sub1_release(addr) \ 469 AO_char_fetch_and_add_release(addr, (unsigned/**/char)(-1)) 470 # define AO_HAVE_char_fetch_and_sub1_release 471 #endif 472 #if defined(AO_HAVE_char_fetch_and_add_acquire) \ 473 && !defined(AO_HAVE_char_fetch_and_sub1_acquire) 474 # define AO_char_fetch_and_sub1_acquire(addr) \ 475 AO_char_fetch_and_add_acquire(addr, (unsigned/**/char)(-1)) 476 # define AO_HAVE_char_fetch_and_sub1_acquire 477 #endif 478 #if defined(AO_HAVE_char_fetch_and_add_write) \ 479 && !defined(AO_HAVE_char_fetch_and_sub1_write) 480 # define AO_char_fetch_and_sub1_write(addr) \ 481 AO_char_fetch_and_add_write(addr, (unsigned/**/char)(-1)) 482 # define AO_HAVE_char_fetch_and_sub1_write 483 #endif 484 #if defined(AO_HAVE_char_fetch_and_add_read) \ 485 && !defined(AO_HAVE_char_fetch_and_sub1_read) 486 # define AO_char_fetch_and_sub1_read(addr) \ 487 AO_char_fetch_and_add_read(addr, (unsigned/**/char)(-1)) 488 # define AO_HAVE_char_fetch_and_sub1_read 489 #endif 490 #if defined(AO_HAVE_char_fetch_and_add_release_write) \ 491 && !defined(AO_HAVE_char_fetch_and_sub1_release_write) 492 # define AO_char_fetch_and_sub1_release_write(addr) \ 493 AO_char_fetch_and_add_release_write(addr, (unsigned/**/char)(-1)) 494 # define AO_HAVE_char_fetch_and_sub1_release_write 495 #endif 496 #if defined(AO_HAVE_char_fetch_and_add_acquire_read) \ 497 && !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) 498 # define AO_char_fetch_and_sub1_acquire_read(addr) \ 499 AO_char_fetch_and_add_acquire_read(addr, (unsigned/**/char)(-1)) 500 # define AO_HAVE_char_fetch_and_sub1_acquire_read 501 #endif 502 #if defined(AO_HAVE_char_fetch_and_add) \ 503 && !defined(AO_HAVE_char_fetch_and_sub1) 504 # define AO_char_fetch_and_sub1(addr) \ 505 AO_char_fetch_and_add(addr, (unsigned/**/char)(-1)) 506 # define AO_HAVE_char_fetch_and_sub1 507 #endif 508 509 #if defined(AO_HAVE_char_fetch_and_sub1_full) 510 # if !defined(AO_HAVE_char_fetch_and_sub1_release) 511 # define AO_char_fetch_and_sub1_release(addr) \ 512 AO_char_fetch_and_sub1_full(addr) 513 # define AO_HAVE_char_fetch_and_sub1_release 514 # endif 515 # if !defined(AO_HAVE_char_fetch_and_sub1_acquire) 516 # define AO_char_fetch_and_sub1_acquire(addr) \ 517 AO_char_fetch_and_sub1_full(addr) 518 # define AO_HAVE_char_fetch_and_sub1_acquire 519 # endif 520 # if !defined(AO_HAVE_char_fetch_and_sub1_write) 521 # define AO_char_fetch_and_sub1_write(addr) \ 522 AO_char_fetch_and_sub1_full(addr) 523 # define AO_HAVE_char_fetch_and_sub1_write 524 # endif 525 # if !defined(AO_HAVE_char_fetch_and_sub1_read) 526 # define AO_char_fetch_and_sub1_read(addr) \ 527 AO_char_fetch_and_sub1_full(addr) 528 # define AO_HAVE_char_fetch_and_sub1_read 529 # endif 530 #endif /* AO_HAVE_char_fetch_and_sub1_full */ 531 532 #if !defined(AO_HAVE_char_fetch_and_sub1) \ 533 && defined(AO_HAVE_char_fetch_and_sub1_release) 534 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_release(addr) 535 # define AO_HAVE_char_fetch_and_sub1 536 #endif 537 #if !defined(AO_HAVE_char_fetch_and_sub1) \ 538 && defined(AO_HAVE_char_fetch_and_sub1_acquire) 539 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_acquire(addr) 540 # define AO_HAVE_char_fetch_and_sub1 541 #endif 542 #if !defined(AO_HAVE_char_fetch_and_sub1) \ 543 && defined(AO_HAVE_char_fetch_and_sub1_write) 544 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_write(addr) 545 # define AO_HAVE_char_fetch_and_sub1 546 #endif 547 #if !defined(AO_HAVE_char_fetch_and_sub1) \ 548 && defined(AO_HAVE_char_fetch_and_sub1_read) 549 # define AO_char_fetch_and_sub1(addr) AO_char_fetch_and_sub1_read(addr) 550 # define AO_HAVE_char_fetch_and_sub1 551 #endif 552 553 #if defined(AO_HAVE_char_fetch_and_sub1_acquire) \ 554 && defined(AO_HAVE_nop_full) \ 555 && !defined(AO_HAVE_char_fetch_and_sub1_full) 556 # define AO_char_fetch_and_sub1_full(addr) \ 557 (AO_nop_full(), AO_char_fetch_and_sub1_acquire(addr)) 558 # define AO_HAVE_char_fetch_and_sub1_full 559 #endif 560 561 #if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \ 562 && defined(AO_HAVE_char_fetch_and_sub1_write) 563 # define AO_char_fetch_and_sub1_release_write(addr) \ 564 AO_char_fetch_and_sub1_write(addr) 565 # define AO_HAVE_char_fetch_and_sub1_release_write 566 #endif 567 #if !defined(AO_HAVE_char_fetch_and_sub1_release_write) \ 568 && defined(AO_HAVE_char_fetch_and_sub1_release) 569 # define AO_char_fetch_and_sub1_release_write(addr) \ 570 AO_char_fetch_and_sub1_release(addr) 571 # define AO_HAVE_char_fetch_and_sub1_release_write 572 #endif 573 #if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \ 574 && defined(AO_HAVE_char_fetch_and_sub1_read) 575 # define AO_char_fetch_and_sub1_acquire_read(addr) \ 576 AO_char_fetch_and_sub1_read(addr) 577 # define AO_HAVE_char_fetch_and_sub1_acquire_read 578 #endif 579 #if !defined(AO_HAVE_char_fetch_and_sub1_acquire_read) \ 580 && defined(AO_HAVE_char_fetch_and_sub1_acquire) 581 # define AO_char_fetch_and_sub1_acquire_read(addr) \ 582 AO_char_fetch_and_sub1_acquire(addr) 583 # define AO_HAVE_char_fetch_and_sub1_acquire_read 584 #endif 585 586 #ifdef AO_NO_DD_ORDERING 587 # if defined(AO_HAVE_char_fetch_and_sub1_acquire_read) 588 # define AO_char_fetch_and_sub1_dd_acquire_read(addr) \ 589 AO_char_fetch_and_sub1_acquire_read(addr) 590 # define AO_HAVE_char_fetch_and_sub1_dd_acquire_read 591 # endif 592 #else 593 # if defined(AO_HAVE_char_fetch_and_sub1) 594 # define AO_char_fetch_and_sub1_dd_acquire_read(addr) \ 595 AO_char_fetch_and_sub1(addr) 596 # define AO_HAVE_char_fetch_and_sub1_dd_acquire_read 597 # endif 598 #endif /* !AO_NO_DD_ORDERING */ 599 600 /* char_and */ 601 #if defined(AO_HAVE_char_compare_and_swap_full) \ 602 && !defined(AO_HAVE_char_and_full) 603 AO_INLINE void AO_char_and_full(volatile unsigned char * addr,unsigned char value)604 AO_char_and_full(volatile unsigned/**/char *addr, unsigned/**/char value) 605 { 606 unsigned/**/char old; 607 608 do 609 { 610 old = *(unsigned/**/char *)addr; 611 } 612 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, 613 old & value))); 614 } 615 # define AO_HAVE_char_and_full 616 #endif 617 618 #if defined(AO_HAVE_char_and_full) 619 # if !defined(AO_HAVE_char_and_release) 620 # define AO_char_and_release(addr, val) AO_char_and_full(addr, val) 621 # define AO_HAVE_char_and_release 622 # endif 623 # if !defined(AO_HAVE_char_and_acquire) 624 # define AO_char_and_acquire(addr, val) AO_char_and_full(addr, val) 625 # define AO_HAVE_char_and_acquire 626 # endif 627 # if !defined(AO_HAVE_char_and_write) 628 # define AO_char_and_write(addr, val) AO_char_and_full(addr, val) 629 # define AO_HAVE_char_and_write 630 # endif 631 # if !defined(AO_HAVE_char_and_read) 632 # define AO_char_and_read(addr, val) AO_char_and_full(addr, val) 633 # define AO_HAVE_char_and_read 634 # endif 635 #endif /* AO_HAVE_char_and_full */ 636 637 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_release) 638 # define AO_char_and(addr, val) AO_char_and_release(addr, val) 639 # define AO_HAVE_char_and 640 #endif 641 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_acquire) 642 # define AO_char_and(addr, val) AO_char_and_acquire(addr, val) 643 # define AO_HAVE_char_and 644 #endif 645 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_write) 646 # define AO_char_and(addr, val) AO_char_and_write(addr, val) 647 # define AO_HAVE_char_and 648 #endif 649 #if !defined(AO_HAVE_char_and) && defined(AO_HAVE_char_and_read) 650 # define AO_char_and(addr, val) AO_char_and_read(addr, val) 651 # define AO_HAVE_char_and 652 #endif 653 654 #if defined(AO_HAVE_char_and_acquire) && defined(AO_HAVE_nop_full) \ 655 && !defined(AO_HAVE_char_and_full) 656 # define AO_char_and_full(addr, val) \ 657 (AO_nop_full(), AO_char_and_acquire(addr, val)) 658 # define AO_HAVE_char_and_full 659 #endif 660 661 #if !defined(AO_HAVE_char_and_release_write) \ 662 && defined(AO_HAVE_char_and_write) 663 # define AO_char_and_release_write(addr, val) AO_char_and_write(addr, val) 664 # define AO_HAVE_char_and_release_write 665 #endif 666 #if !defined(AO_HAVE_char_and_release_write) \ 667 && defined(AO_HAVE_char_and_release) 668 # define AO_char_and_release_write(addr, val) AO_char_and_release(addr, val) 669 # define AO_HAVE_char_and_release_write 670 #endif 671 #if !defined(AO_HAVE_char_and_acquire_read) \ 672 && defined(AO_HAVE_char_and_read) 673 # define AO_char_and_acquire_read(addr, val) AO_char_and_read(addr, val) 674 # define AO_HAVE_char_and_acquire_read 675 #endif 676 #if !defined(AO_HAVE_char_and_acquire_read) \ 677 && defined(AO_HAVE_char_and_acquire) 678 # define AO_char_and_acquire_read(addr, val) AO_char_and_acquire(addr, val) 679 # define AO_HAVE_char_and_acquire_read 680 #endif 681 682 /* char_or */ 683 #if defined(AO_HAVE_char_compare_and_swap_full) \ 684 && !defined(AO_HAVE_char_or_full) 685 AO_INLINE void AO_char_or_full(volatile unsigned char * addr,unsigned char value)686 AO_char_or_full(volatile unsigned/**/char *addr, unsigned/**/char value) 687 { 688 unsigned/**/char old; 689 690 do 691 { 692 old = *(unsigned/**/char *)addr; 693 } 694 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, 695 old | value))); 696 } 697 # define AO_HAVE_char_or_full 698 #endif 699 700 #if defined(AO_HAVE_char_or_full) 701 # if !defined(AO_HAVE_char_or_release) 702 # define AO_char_or_release(addr, val) AO_char_or_full(addr, val) 703 # define AO_HAVE_char_or_release 704 # endif 705 # if !defined(AO_HAVE_char_or_acquire) 706 # define AO_char_or_acquire(addr, val) AO_char_or_full(addr, val) 707 # define AO_HAVE_char_or_acquire 708 # endif 709 # if !defined(AO_HAVE_char_or_write) 710 # define AO_char_or_write(addr, val) AO_char_or_full(addr, val) 711 # define AO_HAVE_char_or_write 712 # endif 713 # if !defined(AO_HAVE_char_or_read) 714 # define AO_char_or_read(addr, val) AO_char_or_full(addr, val) 715 # define AO_HAVE_char_or_read 716 # endif 717 #endif /* AO_HAVE_char_or_full */ 718 719 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_release) 720 # define AO_char_or(addr, val) AO_char_or_release(addr, val) 721 # define AO_HAVE_char_or 722 #endif 723 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_acquire) 724 # define AO_char_or(addr, val) AO_char_or_acquire(addr, val) 725 # define AO_HAVE_char_or 726 #endif 727 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_write) 728 # define AO_char_or(addr, val) AO_char_or_write(addr, val) 729 # define AO_HAVE_char_or 730 #endif 731 #if !defined(AO_HAVE_char_or) && defined(AO_HAVE_char_or_read) 732 # define AO_char_or(addr, val) AO_char_or_read(addr, val) 733 # define AO_HAVE_char_or 734 #endif 735 736 #if defined(AO_HAVE_char_or_acquire) && defined(AO_HAVE_nop_full) \ 737 && !defined(AO_HAVE_char_or_full) 738 # define AO_char_or_full(addr, val) \ 739 (AO_nop_full(), AO_char_or_acquire(addr, val)) 740 # define AO_HAVE_char_or_full 741 #endif 742 743 #if !defined(AO_HAVE_char_or_release_write) \ 744 && defined(AO_HAVE_char_or_write) 745 # define AO_char_or_release_write(addr, val) AO_char_or_write(addr, val) 746 # define AO_HAVE_char_or_release_write 747 #endif 748 #if !defined(AO_HAVE_char_or_release_write) \ 749 && defined(AO_HAVE_char_or_release) 750 # define AO_char_or_release_write(addr, val) AO_char_or_release(addr, val) 751 # define AO_HAVE_char_or_release_write 752 #endif 753 #if !defined(AO_HAVE_char_or_acquire_read) && defined(AO_HAVE_char_or_read) 754 # define AO_char_or_acquire_read(addr, val) AO_char_or_read(addr, val) 755 # define AO_HAVE_char_or_acquire_read 756 #endif 757 #if !defined(AO_HAVE_char_or_acquire_read) \ 758 && defined(AO_HAVE_char_or_acquire) 759 # define AO_char_or_acquire_read(addr, val) AO_char_or_acquire(addr, val) 760 # define AO_HAVE_char_or_acquire_read 761 #endif 762 763 /* char_xor */ 764 #if defined(AO_HAVE_char_compare_and_swap_full) \ 765 && !defined(AO_HAVE_char_xor_full) 766 AO_INLINE void AO_char_xor_full(volatile unsigned char * addr,unsigned char value)767 AO_char_xor_full(volatile unsigned/**/char *addr, unsigned/**/char value) 768 { 769 unsigned/**/char old; 770 771 do 772 { 773 old = *(unsigned/**/char *)addr; 774 } 775 while (AO_EXPECT_FALSE(!AO_char_compare_and_swap_full(addr, old, 776 old ^ value))); 777 } 778 # define AO_HAVE_char_xor_full 779 #endif 780 781 #if defined(AO_HAVE_char_xor_full) 782 # if !defined(AO_HAVE_char_xor_release) 783 # define AO_char_xor_release(addr, val) AO_char_xor_full(addr, val) 784 # define AO_HAVE_char_xor_release 785 # endif 786 # if !defined(AO_HAVE_char_xor_acquire) 787 # define AO_char_xor_acquire(addr, val) AO_char_xor_full(addr, val) 788 # define AO_HAVE_char_xor_acquire 789 # endif 790 # if !defined(AO_HAVE_char_xor_write) 791 # define AO_char_xor_write(addr, val) AO_char_xor_full(addr, val) 792 # define AO_HAVE_char_xor_write 793 # endif 794 # if !defined(AO_HAVE_char_xor_read) 795 # define AO_char_xor_read(addr, val) AO_char_xor_full(addr, val) 796 # define AO_HAVE_char_xor_read 797 # endif 798 #endif /* AO_HAVE_char_xor_full */ 799 800 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_release) 801 # define AO_char_xor(addr, val) AO_char_xor_release(addr, val) 802 # define AO_HAVE_char_xor 803 #endif 804 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_acquire) 805 # define AO_char_xor(addr, val) AO_char_xor_acquire(addr, val) 806 # define AO_HAVE_char_xor 807 #endif 808 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_write) 809 # define AO_char_xor(addr, val) AO_char_xor_write(addr, val) 810 # define AO_HAVE_char_xor 811 #endif 812 #if !defined(AO_HAVE_char_xor) && defined(AO_HAVE_char_xor_read) 813 # define AO_char_xor(addr, val) AO_char_xor_read(addr, val) 814 # define AO_HAVE_char_xor 815 #endif 816 817 #if defined(AO_HAVE_char_xor_acquire) && defined(AO_HAVE_nop_full) \ 818 && !defined(AO_HAVE_char_xor_full) 819 # define AO_char_xor_full(addr, val) \ 820 (AO_nop_full(), AO_char_xor_acquire(addr, val)) 821 # define AO_HAVE_char_xor_full 822 #endif 823 824 #if !defined(AO_HAVE_char_xor_release_write) \ 825 && defined(AO_HAVE_char_xor_write) 826 # define AO_char_xor_release_write(addr, val) AO_char_xor_write(addr, val) 827 # define AO_HAVE_char_xor_release_write 828 #endif 829 #if !defined(AO_HAVE_char_xor_release_write) \ 830 && defined(AO_HAVE_char_xor_release) 831 # define AO_char_xor_release_write(addr, val) AO_char_xor_release(addr, val) 832 # define AO_HAVE_char_xor_release_write 833 #endif 834 #if !defined(AO_HAVE_char_xor_acquire_read) \ 835 && defined(AO_HAVE_char_xor_read) 836 # define AO_char_xor_acquire_read(addr, val) AO_char_xor_read(addr, val) 837 # define AO_HAVE_char_xor_acquire_read 838 #endif 839 #if !defined(AO_HAVE_char_xor_acquire_read) \ 840 && defined(AO_HAVE_char_xor_acquire) 841 # define AO_char_xor_acquire_read(addr, val) AO_char_xor_acquire(addr, val) 842 # define AO_HAVE_char_xor_acquire_read 843 #endif 844 845 /* char_and/or/xor_dd_acquire_read are meaningless. */ 846 /* 847 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. 848 * 849 * Permission is hereby granted, free of charge, to any person obtaining a copy 850 * of this software and associated documentation files (the "Software"), to deal 851 * in the Software without restriction, including without limitation the rights 852 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 853 * copies of the Software, and to permit persons to whom the Software is 854 * furnished to do so, subject to the following conditions: 855 * 856 * The above copyright notice and this permission notice shall be included in 857 * all copies or substantial portions of the Software. 858 * 859 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 860 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 861 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 862 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 863 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 864 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 865 * SOFTWARE. 866 */ 867 868 /* short_compare_and_swap (based on fetch_compare_and_swap) */ 869 #if defined(AO_HAVE_short_fetch_compare_and_swap_full) \ 870 && !defined(AO_HAVE_short_compare_and_swap_full) 871 AO_INLINE int AO_short_compare_and_swap_full(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)872 AO_short_compare_and_swap_full(volatile unsigned/**/short *addr, unsigned/**/short old_val, 873 unsigned/**/short new_val) 874 { 875 return AO_short_fetch_compare_and_swap_full(addr, old_val, new_val) 876 == old_val; 877 } 878 # define AO_HAVE_short_compare_and_swap_full 879 #endif 880 881 #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire) \ 882 && !defined(AO_HAVE_short_compare_and_swap_acquire) 883 AO_INLINE int AO_short_compare_and_swap_acquire(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)884 AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr, unsigned/**/short old_val, 885 unsigned/**/short new_val) 886 { 887 return AO_short_fetch_compare_and_swap_acquire(addr, old_val, new_val) 888 == old_val; 889 } 890 # define AO_HAVE_short_compare_and_swap_acquire 891 #endif 892 893 #if defined(AO_HAVE_short_fetch_compare_and_swap_release) \ 894 && !defined(AO_HAVE_short_compare_and_swap_release) 895 AO_INLINE int AO_short_compare_and_swap_release(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)896 AO_short_compare_and_swap_release(volatile unsigned/**/short *addr, unsigned/**/short old_val, 897 unsigned/**/short new_val) 898 { 899 return AO_short_fetch_compare_and_swap_release(addr, old_val, new_val) 900 == old_val; 901 } 902 # define AO_HAVE_short_compare_and_swap_release 903 #endif 904 905 #if defined(AO_HAVE_short_fetch_compare_and_swap_write) \ 906 && !defined(AO_HAVE_short_compare_and_swap_write) 907 AO_INLINE int AO_short_compare_and_swap_write(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)908 AO_short_compare_and_swap_write(volatile unsigned/**/short *addr, unsigned/**/short old_val, 909 unsigned/**/short new_val) 910 { 911 return AO_short_fetch_compare_and_swap_write(addr, old_val, new_val) 912 == old_val; 913 } 914 # define AO_HAVE_short_compare_and_swap_write 915 #endif 916 917 #if defined(AO_HAVE_short_fetch_compare_and_swap_read) \ 918 && !defined(AO_HAVE_short_compare_and_swap_read) 919 AO_INLINE int AO_short_compare_and_swap_read(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)920 AO_short_compare_and_swap_read(volatile unsigned/**/short *addr, unsigned/**/short old_val, 921 unsigned/**/short new_val) 922 { 923 return AO_short_fetch_compare_and_swap_read(addr, old_val, new_val) 924 == old_val; 925 } 926 # define AO_HAVE_short_compare_and_swap_read 927 #endif 928 929 #if defined(AO_HAVE_short_fetch_compare_and_swap) \ 930 && !defined(AO_HAVE_short_compare_and_swap) 931 AO_INLINE int AO_short_compare_and_swap(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)932 AO_short_compare_and_swap(volatile unsigned/**/short *addr, unsigned/**/short old_val, 933 unsigned/**/short new_val) 934 { 935 return AO_short_fetch_compare_and_swap(addr, old_val, new_val) == old_val; 936 } 937 # define AO_HAVE_short_compare_and_swap 938 #endif 939 940 #if defined(AO_HAVE_short_fetch_compare_and_swap_release_write) \ 941 && !defined(AO_HAVE_short_compare_and_swap_release_write) 942 AO_INLINE int AO_short_compare_and_swap_release_write(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)943 AO_short_compare_and_swap_release_write(volatile unsigned/**/short *addr, 944 unsigned/**/short old_val, unsigned/**/short new_val) 945 { 946 return AO_short_fetch_compare_and_swap_release_write(addr, old_val, 947 new_val) == old_val; 948 } 949 # define AO_HAVE_short_compare_and_swap_release_write 950 #endif 951 952 #if defined(AO_HAVE_short_fetch_compare_and_swap_acquire_read) \ 953 && !defined(AO_HAVE_short_compare_and_swap_acquire_read) 954 AO_INLINE int AO_short_compare_and_swap_acquire_read(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)955 AO_short_compare_and_swap_acquire_read(volatile unsigned/**/short *addr, 956 unsigned/**/short old_val, unsigned/**/short new_val) 957 { 958 return AO_short_fetch_compare_and_swap_acquire_read(addr, old_val, 959 new_val) == old_val; 960 } 961 # define AO_HAVE_short_compare_and_swap_acquire_read 962 #endif 963 964 #if defined(AO_HAVE_short_fetch_compare_and_swap_dd_acquire_read) \ 965 && !defined(AO_HAVE_short_compare_and_swap_dd_acquire_read) 966 AO_INLINE int AO_short_compare_and_swap_dd_acquire_read(volatile unsigned short * addr,unsigned short old_val,unsigned short new_val)967 AO_short_compare_and_swap_dd_acquire_read(volatile unsigned/**/short *addr, 968 unsigned/**/short old_val, unsigned/**/short new_val) 969 { 970 return AO_short_fetch_compare_and_swap_dd_acquire_read(addr, old_val, 971 new_val) == old_val; 972 } 973 # define AO_HAVE_short_compare_and_swap_dd_acquire_read 974 #endif 975 976 /* short_fetch_and_add */ 977 /* We first try to implement fetch_and_add variants in terms of the */ 978 /* corresponding compare_and_swap variants to minimize adding barriers. */ 979 #if defined(AO_HAVE_short_compare_and_swap_full) \ 980 && !defined(AO_HAVE_short_fetch_and_add_full) 981 AO_INLINE unsigned/**/short AO_short_fetch_and_add_full(volatile unsigned short * addr,unsigned short incr)982 AO_short_fetch_and_add_full(volatile unsigned/**/short *addr, unsigned/**/short incr) 983 { 984 unsigned/**/short old; 985 986 do 987 { 988 old = *(unsigned/**/short *)addr; 989 } 990 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, 991 old + incr))); 992 return old; 993 } 994 # define AO_HAVE_short_fetch_and_add_full 995 #endif 996 997 #if defined(AO_HAVE_short_compare_and_swap_acquire) \ 998 && !defined(AO_HAVE_short_fetch_and_add_acquire) 999 AO_INLINE unsigned/**/short AO_short_fetch_and_add_acquire(volatile unsigned short * addr,unsigned short incr)1000 AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr) 1001 { 1002 unsigned/**/short old; 1003 1004 do 1005 { 1006 old = *(unsigned/**/short *)addr; 1007 } 1008 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_acquire(addr, old, 1009 old + incr))); 1010 return old; 1011 } 1012 # define AO_HAVE_short_fetch_and_add_acquire 1013 #endif 1014 1015 #if defined(AO_HAVE_short_compare_and_swap_release) \ 1016 && !defined(AO_HAVE_short_fetch_and_add_release) 1017 AO_INLINE unsigned/**/short AO_short_fetch_and_add_release(volatile unsigned short * addr,unsigned short incr)1018 AO_short_fetch_and_add_release(volatile unsigned/**/short *addr, unsigned/**/short incr) 1019 { 1020 unsigned/**/short old; 1021 1022 do 1023 { 1024 old = *(unsigned/**/short *)addr; 1025 } 1026 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_release(addr, old, 1027 old + incr))); 1028 return old; 1029 } 1030 # define AO_HAVE_short_fetch_and_add_release 1031 #endif 1032 1033 #if defined(AO_HAVE_short_compare_and_swap) \ 1034 && !defined(AO_HAVE_short_fetch_and_add) 1035 AO_INLINE unsigned/**/short AO_short_fetch_and_add(volatile unsigned short * addr,unsigned short incr)1036 AO_short_fetch_and_add(volatile unsigned/**/short *addr, unsigned/**/short incr) 1037 { 1038 unsigned/**/short old; 1039 1040 do 1041 { 1042 old = *(unsigned/**/short *)addr; 1043 } 1044 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap(addr, old, 1045 old + incr))); 1046 return old; 1047 } 1048 # define AO_HAVE_short_fetch_and_add 1049 #endif 1050 1051 #if defined(AO_HAVE_short_fetch_and_add_full) 1052 # if !defined(AO_HAVE_short_fetch_and_add_release) 1053 # define AO_short_fetch_and_add_release(addr, val) \ 1054 AO_short_fetch_and_add_full(addr, val) 1055 # define AO_HAVE_short_fetch_and_add_release 1056 # endif 1057 # if !defined(AO_HAVE_short_fetch_and_add_acquire) 1058 # define AO_short_fetch_and_add_acquire(addr, val) \ 1059 AO_short_fetch_and_add_full(addr, val) 1060 # define AO_HAVE_short_fetch_and_add_acquire 1061 # endif 1062 # if !defined(AO_HAVE_short_fetch_and_add_write) 1063 # define AO_short_fetch_and_add_write(addr, val) \ 1064 AO_short_fetch_and_add_full(addr, val) 1065 # define AO_HAVE_short_fetch_and_add_write 1066 # endif 1067 # if !defined(AO_HAVE_short_fetch_and_add_read) 1068 # define AO_short_fetch_and_add_read(addr, val) \ 1069 AO_short_fetch_and_add_full(addr, val) 1070 # define AO_HAVE_short_fetch_and_add_read 1071 # endif 1072 #endif /* AO_HAVE_short_fetch_and_add_full */ 1073 1074 #if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \ 1075 && !defined(AO_HAVE_short_fetch_and_add_acquire) 1076 AO_INLINE unsigned/**/short AO_short_fetch_and_add_acquire(volatile unsigned short * addr,unsigned short incr)1077 AO_short_fetch_and_add_acquire(volatile unsigned/**/short *addr, unsigned/**/short incr) 1078 { 1079 unsigned/**/short result = AO_short_fetch_and_add(addr, incr); 1080 AO_nop_full(); 1081 return result; 1082 } 1083 # define AO_HAVE_short_fetch_and_add_acquire 1084 #endif 1085 #if defined(AO_HAVE_short_fetch_and_add) && defined(AO_HAVE_nop_full) \ 1086 && !defined(AO_HAVE_short_fetch_and_add_release) 1087 # define AO_short_fetch_and_add_release(addr, incr) \ 1088 (AO_nop_full(), AO_short_fetch_and_add(addr, incr)) 1089 # define AO_HAVE_short_fetch_and_add_release 1090 #endif 1091 1092 #if !defined(AO_HAVE_short_fetch_and_add) \ 1093 && defined(AO_HAVE_short_fetch_and_add_release) 1094 # define AO_short_fetch_and_add(addr, val) \ 1095 AO_short_fetch_and_add_release(addr, val) 1096 # define AO_HAVE_short_fetch_and_add 1097 #endif 1098 #if !defined(AO_HAVE_short_fetch_and_add) \ 1099 && defined(AO_HAVE_short_fetch_and_add_acquire) 1100 # define AO_short_fetch_and_add(addr, val) \ 1101 AO_short_fetch_and_add_acquire(addr, val) 1102 # define AO_HAVE_short_fetch_and_add 1103 #endif 1104 #if !defined(AO_HAVE_short_fetch_and_add) \ 1105 && defined(AO_HAVE_short_fetch_and_add_write) 1106 # define AO_short_fetch_and_add(addr, val) \ 1107 AO_short_fetch_and_add_write(addr, val) 1108 # define AO_HAVE_short_fetch_and_add 1109 #endif 1110 #if !defined(AO_HAVE_short_fetch_and_add) \ 1111 && defined(AO_HAVE_short_fetch_and_add_read) 1112 # define AO_short_fetch_and_add(addr, val) \ 1113 AO_short_fetch_and_add_read(addr, val) 1114 # define AO_HAVE_short_fetch_and_add 1115 #endif 1116 1117 #if defined(AO_HAVE_short_fetch_and_add_acquire) \ 1118 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_short_fetch_and_add_full) 1119 # define AO_short_fetch_and_add_full(addr, val) \ 1120 (AO_nop_full(), AO_short_fetch_and_add_acquire(addr, val)) 1121 # define AO_HAVE_short_fetch_and_add_full 1122 #endif 1123 1124 #if !defined(AO_HAVE_short_fetch_and_add_release_write) \ 1125 && defined(AO_HAVE_short_fetch_and_add_write) 1126 # define AO_short_fetch_and_add_release_write(addr, val) \ 1127 AO_short_fetch_and_add_write(addr, val) 1128 # define AO_HAVE_short_fetch_and_add_release_write 1129 #endif 1130 #if !defined(AO_HAVE_short_fetch_and_add_release_write) \ 1131 && defined(AO_HAVE_short_fetch_and_add_release) 1132 # define AO_short_fetch_and_add_release_write(addr, val) \ 1133 AO_short_fetch_and_add_release(addr, val) 1134 # define AO_HAVE_short_fetch_and_add_release_write 1135 #endif 1136 1137 #if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \ 1138 && defined(AO_HAVE_short_fetch_and_add_read) 1139 # define AO_short_fetch_and_add_acquire_read(addr, val) \ 1140 AO_short_fetch_and_add_read(addr, val) 1141 # define AO_HAVE_short_fetch_and_add_acquire_read 1142 #endif 1143 #if !defined(AO_HAVE_short_fetch_and_add_acquire_read) \ 1144 && defined(AO_HAVE_short_fetch_and_add_acquire) 1145 # define AO_short_fetch_and_add_acquire_read(addr, val) \ 1146 AO_short_fetch_and_add_acquire(addr, val) 1147 # define AO_HAVE_short_fetch_and_add_acquire_read 1148 #endif 1149 1150 #ifdef AO_NO_DD_ORDERING 1151 # if defined(AO_HAVE_short_fetch_and_add_acquire_read) 1152 # define AO_short_fetch_and_add_dd_acquire_read(addr, val) \ 1153 AO_short_fetch_and_add_acquire_read(addr, val) 1154 # define AO_HAVE_short_fetch_and_add_dd_acquire_read 1155 # endif 1156 #else 1157 # if defined(AO_HAVE_short_fetch_and_add) 1158 # define AO_short_fetch_and_add_dd_acquire_read(addr, val) \ 1159 AO_short_fetch_and_add(addr, val) 1160 # define AO_HAVE_short_fetch_and_add_dd_acquire_read 1161 # endif 1162 #endif /* !AO_NO_DD_ORDERING */ 1163 1164 /* short_fetch_and_add1 */ 1165 #if defined(AO_HAVE_short_fetch_and_add_full) \ 1166 && !defined(AO_HAVE_short_fetch_and_add1_full) 1167 # define AO_short_fetch_and_add1_full(addr) \ 1168 AO_short_fetch_and_add_full(addr, 1) 1169 # define AO_HAVE_short_fetch_and_add1_full 1170 #endif 1171 #if defined(AO_HAVE_short_fetch_and_add_release) \ 1172 && !defined(AO_HAVE_short_fetch_and_add1_release) 1173 # define AO_short_fetch_and_add1_release(addr) \ 1174 AO_short_fetch_and_add_release(addr, 1) 1175 # define AO_HAVE_short_fetch_and_add1_release 1176 #endif 1177 #if defined(AO_HAVE_short_fetch_and_add_acquire) \ 1178 && !defined(AO_HAVE_short_fetch_and_add1_acquire) 1179 # define AO_short_fetch_and_add1_acquire(addr) \ 1180 AO_short_fetch_and_add_acquire(addr, 1) 1181 # define AO_HAVE_short_fetch_and_add1_acquire 1182 #endif 1183 #if defined(AO_HAVE_short_fetch_and_add_write) \ 1184 && !defined(AO_HAVE_short_fetch_and_add1_write) 1185 # define AO_short_fetch_and_add1_write(addr) \ 1186 AO_short_fetch_and_add_write(addr, 1) 1187 # define AO_HAVE_short_fetch_and_add1_write 1188 #endif 1189 #if defined(AO_HAVE_short_fetch_and_add_read) \ 1190 && !defined(AO_HAVE_short_fetch_and_add1_read) 1191 # define AO_short_fetch_and_add1_read(addr) \ 1192 AO_short_fetch_and_add_read(addr, 1) 1193 # define AO_HAVE_short_fetch_and_add1_read 1194 #endif 1195 #if defined(AO_HAVE_short_fetch_and_add_release_write) \ 1196 && !defined(AO_HAVE_short_fetch_and_add1_release_write) 1197 # define AO_short_fetch_and_add1_release_write(addr) \ 1198 AO_short_fetch_and_add_release_write(addr, 1) 1199 # define AO_HAVE_short_fetch_and_add1_release_write 1200 #endif 1201 #if defined(AO_HAVE_short_fetch_and_add_acquire_read) \ 1202 && !defined(AO_HAVE_short_fetch_and_add1_acquire_read) 1203 # define AO_short_fetch_and_add1_acquire_read(addr) \ 1204 AO_short_fetch_and_add_acquire_read(addr, 1) 1205 # define AO_HAVE_short_fetch_and_add1_acquire_read 1206 #endif 1207 #if defined(AO_HAVE_short_fetch_and_add) \ 1208 && !defined(AO_HAVE_short_fetch_and_add1) 1209 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add(addr, 1) 1210 # define AO_HAVE_short_fetch_and_add1 1211 #endif 1212 1213 #if defined(AO_HAVE_short_fetch_and_add1_full) 1214 # if !defined(AO_HAVE_short_fetch_and_add1_release) 1215 # define AO_short_fetch_and_add1_release(addr) \ 1216 AO_short_fetch_and_add1_full(addr) 1217 # define AO_HAVE_short_fetch_and_add1_release 1218 # endif 1219 # if !defined(AO_HAVE_short_fetch_and_add1_acquire) 1220 # define AO_short_fetch_and_add1_acquire(addr) \ 1221 AO_short_fetch_and_add1_full(addr) 1222 # define AO_HAVE_short_fetch_and_add1_acquire 1223 # endif 1224 # if !defined(AO_HAVE_short_fetch_and_add1_write) 1225 # define AO_short_fetch_and_add1_write(addr) \ 1226 AO_short_fetch_and_add1_full(addr) 1227 # define AO_HAVE_short_fetch_and_add1_write 1228 # endif 1229 # if !defined(AO_HAVE_short_fetch_and_add1_read) 1230 # define AO_short_fetch_and_add1_read(addr) \ 1231 AO_short_fetch_and_add1_full(addr) 1232 # define AO_HAVE_short_fetch_and_add1_read 1233 # endif 1234 #endif /* AO_HAVE_short_fetch_and_add1_full */ 1235 1236 #if !defined(AO_HAVE_short_fetch_and_add1) \ 1237 && defined(AO_HAVE_short_fetch_and_add1_release) 1238 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_release(addr) 1239 # define AO_HAVE_short_fetch_and_add1 1240 #endif 1241 #if !defined(AO_HAVE_short_fetch_and_add1) \ 1242 && defined(AO_HAVE_short_fetch_and_add1_acquire) 1243 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_acquire(addr) 1244 # define AO_HAVE_short_fetch_and_add1 1245 #endif 1246 #if !defined(AO_HAVE_short_fetch_and_add1) \ 1247 && defined(AO_HAVE_short_fetch_and_add1_write) 1248 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_write(addr) 1249 # define AO_HAVE_short_fetch_and_add1 1250 #endif 1251 #if !defined(AO_HAVE_short_fetch_and_add1) \ 1252 && defined(AO_HAVE_short_fetch_and_add1_read) 1253 # define AO_short_fetch_and_add1(addr) AO_short_fetch_and_add1_read(addr) 1254 # define AO_HAVE_short_fetch_and_add1 1255 #endif 1256 1257 #if defined(AO_HAVE_short_fetch_and_add1_acquire) \ 1258 && defined(AO_HAVE_nop_full) \ 1259 && !defined(AO_HAVE_short_fetch_and_add1_full) 1260 # define AO_short_fetch_and_add1_full(addr) \ 1261 (AO_nop_full(), AO_short_fetch_and_add1_acquire(addr)) 1262 # define AO_HAVE_short_fetch_and_add1_full 1263 #endif 1264 1265 #if !defined(AO_HAVE_short_fetch_and_add1_release_write) \ 1266 && defined(AO_HAVE_short_fetch_and_add1_write) 1267 # define AO_short_fetch_and_add1_release_write(addr) \ 1268 AO_short_fetch_and_add1_write(addr) 1269 # define AO_HAVE_short_fetch_and_add1_release_write 1270 #endif 1271 #if !defined(AO_HAVE_short_fetch_and_add1_release_write) \ 1272 && defined(AO_HAVE_short_fetch_and_add1_release) 1273 # define AO_short_fetch_and_add1_release_write(addr) \ 1274 AO_short_fetch_and_add1_release(addr) 1275 # define AO_HAVE_short_fetch_and_add1_release_write 1276 #endif 1277 #if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \ 1278 && defined(AO_HAVE_short_fetch_and_add1_read) 1279 # define AO_short_fetch_and_add1_acquire_read(addr) \ 1280 AO_short_fetch_and_add1_read(addr) 1281 # define AO_HAVE_short_fetch_and_add1_acquire_read 1282 #endif 1283 #if !defined(AO_HAVE_short_fetch_and_add1_acquire_read) \ 1284 && defined(AO_HAVE_short_fetch_and_add1_acquire) 1285 # define AO_short_fetch_and_add1_acquire_read(addr) \ 1286 AO_short_fetch_and_add1_acquire(addr) 1287 # define AO_HAVE_short_fetch_and_add1_acquire_read 1288 #endif 1289 1290 #ifdef AO_NO_DD_ORDERING 1291 # if defined(AO_HAVE_short_fetch_and_add1_acquire_read) 1292 # define AO_short_fetch_and_add1_dd_acquire_read(addr) \ 1293 AO_short_fetch_and_add1_acquire_read(addr) 1294 # define AO_HAVE_short_fetch_and_add1_dd_acquire_read 1295 # endif 1296 #else 1297 # if defined(AO_HAVE_short_fetch_and_add1) 1298 # define AO_short_fetch_and_add1_dd_acquire_read(addr) \ 1299 AO_short_fetch_and_add1(addr) 1300 # define AO_HAVE_short_fetch_and_add1_dd_acquire_read 1301 # endif 1302 #endif /* !AO_NO_DD_ORDERING */ 1303 1304 /* short_fetch_and_sub1 */ 1305 #if defined(AO_HAVE_short_fetch_and_add_full) \ 1306 && !defined(AO_HAVE_short_fetch_and_sub1_full) 1307 # define AO_short_fetch_and_sub1_full(addr) \ 1308 AO_short_fetch_and_add_full(addr, (unsigned/**/short)(-1)) 1309 # define AO_HAVE_short_fetch_and_sub1_full 1310 #endif 1311 #if defined(AO_HAVE_short_fetch_and_add_release) \ 1312 && !defined(AO_HAVE_short_fetch_and_sub1_release) 1313 # define AO_short_fetch_and_sub1_release(addr) \ 1314 AO_short_fetch_and_add_release(addr, (unsigned/**/short)(-1)) 1315 # define AO_HAVE_short_fetch_and_sub1_release 1316 #endif 1317 #if defined(AO_HAVE_short_fetch_and_add_acquire) \ 1318 && !defined(AO_HAVE_short_fetch_and_sub1_acquire) 1319 # define AO_short_fetch_and_sub1_acquire(addr) \ 1320 AO_short_fetch_and_add_acquire(addr, (unsigned/**/short)(-1)) 1321 # define AO_HAVE_short_fetch_and_sub1_acquire 1322 #endif 1323 #if defined(AO_HAVE_short_fetch_and_add_write) \ 1324 && !defined(AO_HAVE_short_fetch_and_sub1_write) 1325 # define AO_short_fetch_and_sub1_write(addr) \ 1326 AO_short_fetch_and_add_write(addr, (unsigned/**/short)(-1)) 1327 # define AO_HAVE_short_fetch_and_sub1_write 1328 #endif 1329 #if defined(AO_HAVE_short_fetch_and_add_read) \ 1330 && !defined(AO_HAVE_short_fetch_and_sub1_read) 1331 # define AO_short_fetch_and_sub1_read(addr) \ 1332 AO_short_fetch_and_add_read(addr, (unsigned/**/short)(-1)) 1333 # define AO_HAVE_short_fetch_and_sub1_read 1334 #endif 1335 #if defined(AO_HAVE_short_fetch_and_add_release_write) \ 1336 && !defined(AO_HAVE_short_fetch_and_sub1_release_write) 1337 # define AO_short_fetch_and_sub1_release_write(addr) \ 1338 AO_short_fetch_and_add_release_write(addr, (unsigned/**/short)(-1)) 1339 # define AO_HAVE_short_fetch_and_sub1_release_write 1340 #endif 1341 #if defined(AO_HAVE_short_fetch_and_add_acquire_read) \ 1342 && !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) 1343 # define AO_short_fetch_and_sub1_acquire_read(addr) \ 1344 AO_short_fetch_and_add_acquire_read(addr, (unsigned/**/short)(-1)) 1345 # define AO_HAVE_short_fetch_and_sub1_acquire_read 1346 #endif 1347 #if defined(AO_HAVE_short_fetch_and_add) \ 1348 && !defined(AO_HAVE_short_fetch_and_sub1) 1349 # define AO_short_fetch_and_sub1(addr) \ 1350 AO_short_fetch_and_add(addr, (unsigned/**/short)(-1)) 1351 # define AO_HAVE_short_fetch_and_sub1 1352 #endif 1353 1354 #if defined(AO_HAVE_short_fetch_and_sub1_full) 1355 # if !defined(AO_HAVE_short_fetch_and_sub1_release) 1356 # define AO_short_fetch_and_sub1_release(addr) \ 1357 AO_short_fetch_and_sub1_full(addr) 1358 # define AO_HAVE_short_fetch_and_sub1_release 1359 # endif 1360 # if !defined(AO_HAVE_short_fetch_and_sub1_acquire) 1361 # define AO_short_fetch_and_sub1_acquire(addr) \ 1362 AO_short_fetch_and_sub1_full(addr) 1363 # define AO_HAVE_short_fetch_and_sub1_acquire 1364 # endif 1365 # if !defined(AO_HAVE_short_fetch_and_sub1_write) 1366 # define AO_short_fetch_and_sub1_write(addr) \ 1367 AO_short_fetch_and_sub1_full(addr) 1368 # define AO_HAVE_short_fetch_and_sub1_write 1369 # endif 1370 # if !defined(AO_HAVE_short_fetch_and_sub1_read) 1371 # define AO_short_fetch_and_sub1_read(addr) \ 1372 AO_short_fetch_and_sub1_full(addr) 1373 # define AO_HAVE_short_fetch_and_sub1_read 1374 # endif 1375 #endif /* AO_HAVE_short_fetch_and_sub1_full */ 1376 1377 #if !defined(AO_HAVE_short_fetch_and_sub1) \ 1378 && defined(AO_HAVE_short_fetch_and_sub1_release) 1379 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_release(addr) 1380 # define AO_HAVE_short_fetch_and_sub1 1381 #endif 1382 #if !defined(AO_HAVE_short_fetch_and_sub1) \ 1383 && defined(AO_HAVE_short_fetch_and_sub1_acquire) 1384 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_acquire(addr) 1385 # define AO_HAVE_short_fetch_and_sub1 1386 #endif 1387 #if !defined(AO_HAVE_short_fetch_and_sub1) \ 1388 && defined(AO_HAVE_short_fetch_and_sub1_write) 1389 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_write(addr) 1390 # define AO_HAVE_short_fetch_and_sub1 1391 #endif 1392 #if !defined(AO_HAVE_short_fetch_and_sub1) \ 1393 && defined(AO_HAVE_short_fetch_and_sub1_read) 1394 # define AO_short_fetch_and_sub1(addr) AO_short_fetch_and_sub1_read(addr) 1395 # define AO_HAVE_short_fetch_and_sub1 1396 #endif 1397 1398 #if defined(AO_HAVE_short_fetch_and_sub1_acquire) \ 1399 && defined(AO_HAVE_nop_full) \ 1400 && !defined(AO_HAVE_short_fetch_and_sub1_full) 1401 # define AO_short_fetch_and_sub1_full(addr) \ 1402 (AO_nop_full(), AO_short_fetch_and_sub1_acquire(addr)) 1403 # define AO_HAVE_short_fetch_and_sub1_full 1404 #endif 1405 1406 #if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \ 1407 && defined(AO_HAVE_short_fetch_and_sub1_write) 1408 # define AO_short_fetch_and_sub1_release_write(addr) \ 1409 AO_short_fetch_and_sub1_write(addr) 1410 # define AO_HAVE_short_fetch_and_sub1_release_write 1411 #endif 1412 #if !defined(AO_HAVE_short_fetch_and_sub1_release_write) \ 1413 && defined(AO_HAVE_short_fetch_and_sub1_release) 1414 # define AO_short_fetch_and_sub1_release_write(addr) \ 1415 AO_short_fetch_and_sub1_release(addr) 1416 # define AO_HAVE_short_fetch_and_sub1_release_write 1417 #endif 1418 #if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \ 1419 && defined(AO_HAVE_short_fetch_and_sub1_read) 1420 # define AO_short_fetch_and_sub1_acquire_read(addr) \ 1421 AO_short_fetch_and_sub1_read(addr) 1422 # define AO_HAVE_short_fetch_and_sub1_acquire_read 1423 #endif 1424 #if !defined(AO_HAVE_short_fetch_and_sub1_acquire_read) \ 1425 && defined(AO_HAVE_short_fetch_and_sub1_acquire) 1426 # define AO_short_fetch_and_sub1_acquire_read(addr) \ 1427 AO_short_fetch_and_sub1_acquire(addr) 1428 # define AO_HAVE_short_fetch_and_sub1_acquire_read 1429 #endif 1430 1431 #ifdef AO_NO_DD_ORDERING 1432 # if defined(AO_HAVE_short_fetch_and_sub1_acquire_read) 1433 # define AO_short_fetch_and_sub1_dd_acquire_read(addr) \ 1434 AO_short_fetch_and_sub1_acquire_read(addr) 1435 # define AO_HAVE_short_fetch_and_sub1_dd_acquire_read 1436 # endif 1437 #else 1438 # if defined(AO_HAVE_short_fetch_and_sub1) 1439 # define AO_short_fetch_and_sub1_dd_acquire_read(addr) \ 1440 AO_short_fetch_and_sub1(addr) 1441 # define AO_HAVE_short_fetch_and_sub1_dd_acquire_read 1442 # endif 1443 #endif /* !AO_NO_DD_ORDERING */ 1444 1445 /* short_and */ 1446 #if defined(AO_HAVE_short_compare_and_swap_full) \ 1447 && !defined(AO_HAVE_short_and_full) 1448 AO_INLINE void AO_short_and_full(volatile unsigned short * addr,unsigned short value)1449 AO_short_and_full(volatile unsigned/**/short *addr, unsigned/**/short value) 1450 { 1451 unsigned/**/short old; 1452 1453 do 1454 { 1455 old = *(unsigned/**/short *)addr; 1456 } 1457 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, 1458 old & value))); 1459 } 1460 # define AO_HAVE_short_and_full 1461 #endif 1462 1463 #if defined(AO_HAVE_short_and_full) 1464 # if !defined(AO_HAVE_short_and_release) 1465 # define AO_short_and_release(addr, val) AO_short_and_full(addr, val) 1466 # define AO_HAVE_short_and_release 1467 # endif 1468 # if !defined(AO_HAVE_short_and_acquire) 1469 # define AO_short_and_acquire(addr, val) AO_short_and_full(addr, val) 1470 # define AO_HAVE_short_and_acquire 1471 # endif 1472 # if !defined(AO_HAVE_short_and_write) 1473 # define AO_short_and_write(addr, val) AO_short_and_full(addr, val) 1474 # define AO_HAVE_short_and_write 1475 # endif 1476 # if !defined(AO_HAVE_short_and_read) 1477 # define AO_short_and_read(addr, val) AO_short_and_full(addr, val) 1478 # define AO_HAVE_short_and_read 1479 # endif 1480 #endif /* AO_HAVE_short_and_full */ 1481 1482 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_release) 1483 # define AO_short_and(addr, val) AO_short_and_release(addr, val) 1484 # define AO_HAVE_short_and 1485 #endif 1486 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_acquire) 1487 # define AO_short_and(addr, val) AO_short_and_acquire(addr, val) 1488 # define AO_HAVE_short_and 1489 #endif 1490 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_write) 1491 # define AO_short_and(addr, val) AO_short_and_write(addr, val) 1492 # define AO_HAVE_short_and 1493 #endif 1494 #if !defined(AO_HAVE_short_and) && defined(AO_HAVE_short_and_read) 1495 # define AO_short_and(addr, val) AO_short_and_read(addr, val) 1496 # define AO_HAVE_short_and 1497 #endif 1498 1499 #if defined(AO_HAVE_short_and_acquire) && defined(AO_HAVE_nop_full) \ 1500 && !defined(AO_HAVE_short_and_full) 1501 # define AO_short_and_full(addr, val) \ 1502 (AO_nop_full(), AO_short_and_acquire(addr, val)) 1503 # define AO_HAVE_short_and_full 1504 #endif 1505 1506 #if !defined(AO_HAVE_short_and_release_write) \ 1507 && defined(AO_HAVE_short_and_write) 1508 # define AO_short_and_release_write(addr, val) AO_short_and_write(addr, val) 1509 # define AO_HAVE_short_and_release_write 1510 #endif 1511 #if !defined(AO_HAVE_short_and_release_write) \ 1512 && defined(AO_HAVE_short_and_release) 1513 # define AO_short_and_release_write(addr, val) AO_short_and_release(addr, val) 1514 # define AO_HAVE_short_and_release_write 1515 #endif 1516 #if !defined(AO_HAVE_short_and_acquire_read) \ 1517 && defined(AO_HAVE_short_and_read) 1518 # define AO_short_and_acquire_read(addr, val) AO_short_and_read(addr, val) 1519 # define AO_HAVE_short_and_acquire_read 1520 #endif 1521 #if !defined(AO_HAVE_short_and_acquire_read) \ 1522 && defined(AO_HAVE_short_and_acquire) 1523 # define AO_short_and_acquire_read(addr, val) AO_short_and_acquire(addr, val) 1524 # define AO_HAVE_short_and_acquire_read 1525 #endif 1526 1527 /* short_or */ 1528 #if defined(AO_HAVE_short_compare_and_swap_full) \ 1529 && !defined(AO_HAVE_short_or_full) 1530 AO_INLINE void AO_short_or_full(volatile unsigned short * addr,unsigned short value)1531 AO_short_or_full(volatile unsigned/**/short *addr, unsigned/**/short value) 1532 { 1533 unsigned/**/short old; 1534 1535 do 1536 { 1537 old = *(unsigned/**/short *)addr; 1538 } 1539 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, 1540 old | value))); 1541 } 1542 # define AO_HAVE_short_or_full 1543 #endif 1544 1545 #if defined(AO_HAVE_short_or_full) 1546 # if !defined(AO_HAVE_short_or_release) 1547 # define AO_short_or_release(addr, val) AO_short_or_full(addr, val) 1548 # define AO_HAVE_short_or_release 1549 # endif 1550 # if !defined(AO_HAVE_short_or_acquire) 1551 # define AO_short_or_acquire(addr, val) AO_short_or_full(addr, val) 1552 # define AO_HAVE_short_or_acquire 1553 # endif 1554 # if !defined(AO_HAVE_short_or_write) 1555 # define AO_short_or_write(addr, val) AO_short_or_full(addr, val) 1556 # define AO_HAVE_short_or_write 1557 # endif 1558 # if !defined(AO_HAVE_short_or_read) 1559 # define AO_short_or_read(addr, val) AO_short_or_full(addr, val) 1560 # define AO_HAVE_short_or_read 1561 # endif 1562 #endif /* AO_HAVE_short_or_full */ 1563 1564 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_release) 1565 # define AO_short_or(addr, val) AO_short_or_release(addr, val) 1566 # define AO_HAVE_short_or 1567 #endif 1568 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_acquire) 1569 # define AO_short_or(addr, val) AO_short_or_acquire(addr, val) 1570 # define AO_HAVE_short_or 1571 #endif 1572 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_write) 1573 # define AO_short_or(addr, val) AO_short_or_write(addr, val) 1574 # define AO_HAVE_short_or 1575 #endif 1576 #if !defined(AO_HAVE_short_or) && defined(AO_HAVE_short_or_read) 1577 # define AO_short_or(addr, val) AO_short_or_read(addr, val) 1578 # define AO_HAVE_short_or 1579 #endif 1580 1581 #if defined(AO_HAVE_short_or_acquire) && defined(AO_HAVE_nop_full) \ 1582 && !defined(AO_HAVE_short_or_full) 1583 # define AO_short_or_full(addr, val) \ 1584 (AO_nop_full(), AO_short_or_acquire(addr, val)) 1585 # define AO_HAVE_short_or_full 1586 #endif 1587 1588 #if !defined(AO_HAVE_short_or_release_write) \ 1589 && defined(AO_HAVE_short_or_write) 1590 # define AO_short_or_release_write(addr, val) AO_short_or_write(addr, val) 1591 # define AO_HAVE_short_or_release_write 1592 #endif 1593 #if !defined(AO_HAVE_short_or_release_write) \ 1594 && defined(AO_HAVE_short_or_release) 1595 # define AO_short_or_release_write(addr, val) AO_short_or_release(addr, val) 1596 # define AO_HAVE_short_or_release_write 1597 #endif 1598 #if !defined(AO_HAVE_short_or_acquire_read) && defined(AO_HAVE_short_or_read) 1599 # define AO_short_or_acquire_read(addr, val) AO_short_or_read(addr, val) 1600 # define AO_HAVE_short_or_acquire_read 1601 #endif 1602 #if !defined(AO_HAVE_short_or_acquire_read) \ 1603 && defined(AO_HAVE_short_or_acquire) 1604 # define AO_short_or_acquire_read(addr, val) AO_short_or_acquire(addr, val) 1605 # define AO_HAVE_short_or_acquire_read 1606 #endif 1607 1608 /* short_xor */ 1609 #if defined(AO_HAVE_short_compare_and_swap_full) \ 1610 && !defined(AO_HAVE_short_xor_full) 1611 AO_INLINE void AO_short_xor_full(volatile unsigned short * addr,unsigned short value)1612 AO_short_xor_full(volatile unsigned/**/short *addr, unsigned/**/short value) 1613 { 1614 unsigned/**/short old; 1615 1616 do 1617 { 1618 old = *(unsigned/**/short *)addr; 1619 } 1620 while (AO_EXPECT_FALSE(!AO_short_compare_and_swap_full(addr, old, 1621 old ^ value))); 1622 } 1623 # define AO_HAVE_short_xor_full 1624 #endif 1625 1626 #if defined(AO_HAVE_short_xor_full) 1627 # if !defined(AO_HAVE_short_xor_release) 1628 # define AO_short_xor_release(addr, val) AO_short_xor_full(addr, val) 1629 # define AO_HAVE_short_xor_release 1630 # endif 1631 # if !defined(AO_HAVE_short_xor_acquire) 1632 # define AO_short_xor_acquire(addr, val) AO_short_xor_full(addr, val) 1633 # define AO_HAVE_short_xor_acquire 1634 # endif 1635 # if !defined(AO_HAVE_short_xor_write) 1636 # define AO_short_xor_write(addr, val) AO_short_xor_full(addr, val) 1637 # define AO_HAVE_short_xor_write 1638 # endif 1639 # if !defined(AO_HAVE_short_xor_read) 1640 # define AO_short_xor_read(addr, val) AO_short_xor_full(addr, val) 1641 # define AO_HAVE_short_xor_read 1642 # endif 1643 #endif /* AO_HAVE_short_xor_full */ 1644 1645 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_release) 1646 # define AO_short_xor(addr, val) AO_short_xor_release(addr, val) 1647 # define AO_HAVE_short_xor 1648 #endif 1649 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_acquire) 1650 # define AO_short_xor(addr, val) AO_short_xor_acquire(addr, val) 1651 # define AO_HAVE_short_xor 1652 #endif 1653 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_write) 1654 # define AO_short_xor(addr, val) AO_short_xor_write(addr, val) 1655 # define AO_HAVE_short_xor 1656 #endif 1657 #if !defined(AO_HAVE_short_xor) && defined(AO_HAVE_short_xor_read) 1658 # define AO_short_xor(addr, val) AO_short_xor_read(addr, val) 1659 # define AO_HAVE_short_xor 1660 #endif 1661 1662 #if defined(AO_HAVE_short_xor_acquire) && defined(AO_HAVE_nop_full) \ 1663 && !defined(AO_HAVE_short_xor_full) 1664 # define AO_short_xor_full(addr, val) \ 1665 (AO_nop_full(), AO_short_xor_acquire(addr, val)) 1666 # define AO_HAVE_short_xor_full 1667 #endif 1668 1669 #if !defined(AO_HAVE_short_xor_release_write) \ 1670 && defined(AO_HAVE_short_xor_write) 1671 # define AO_short_xor_release_write(addr, val) AO_short_xor_write(addr, val) 1672 # define AO_HAVE_short_xor_release_write 1673 #endif 1674 #if !defined(AO_HAVE_short_xor_release_write) \ 1675 && defined(AO_HAVE_short_xor_release) 1676 # define AO_short_xor_release_write(addr, val) AO_short_xor_release(addr, val) 1677 # define AO_HAVE_short_xor_release_write 1678 #endif 1679 #if !defined(AO_HAVE_short_xor_acquire_read) \ 1680 && defined(AO_HAVE_short_xor_read) 1681 # define AO_short_xor_acquire_read(addr, val) AO_short_xor_read(addr, val) 1682 # define AO_HAVE_short_xor_acquire_read 1683 #endif 1684 #if !defined(AO_HAVE_short_xor_acquire_read) \ 1685 && defined(AO_HAVE_short_xor_acquire) 1686 # define AO_short_xor_acquire_read(addr, val) AO_short_xor_acquire(addr, val) 1687 # define AO_HAVE_short_xor_acquire_read 1688 #endif 1689 1690 /* short_and/or/xor_dd_acquire_read are meaningless. */ 1691 /* 1692 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. 1693 * 1694 * Permission is hereby granted, free of charge, to any person obtaining a copy 1695 * of this software and associated documentation files (the "Software"), to deal 1696 * in the Software without restriction, including without limitation the rights 1697 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 1698 * copies of the Software, and to permit persons to whom the Software is 1699 * furnished to do so, subject to the following conditions: 1700 * 1701 * The above copyright notice and this permission notice shall be included in 1702 * all copies or substantial portions of the Software. 1703 * 1704 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1705 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1706 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 1707 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 1708 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 1709 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 1710 * SOFTWARE. 1711 */ 1712 1713 /* int_compare_and_swap (based on fetch_compare_and_swap) */ 1714 #if defined(AO_HAVE_int_fetch_compare_and_swap_full) \ 1715 && !defined(AO_HAVE_int_compare_and_swap_full) 1716 AO_INLINE int AO_int_compare_and_swap_full(volatile unsigned * addr,unsigned old_val,unsigned new_val)1717 AO_int_compare_and_swap_full(volatile unsigned *addr, unsigned old_val, 1718 unsigned new_val) 1719 { 1720 return AO_int_fetch_compare_and_swap_full(addr, old_val, new_val) 1721 == old_val; 1722 } 1723 # define AO_HAVE_int_compare_and_swap_full 1724 #endif 1725 1726 #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire) \ 1727 && !defined(AO_HAVE_int_compare_and_swap_acquire) 1728 AO_INLINE int AO_int_compare_and_swap_acquire(volatile unsigned * addr,unsigned old_val,unsigned new_val)1729 AO_int_compare_and_swap_acquire(volatile unsigned *addr, unsigned old_val, 1730 unsigned new_val) 1731 { 1732 return AO_int_fetch_compare_and_swap_acquire(addr, old_val, new_val) 1733 == old_val; 1734 } 1735 # define AO_HAVE_int_compare_and_swap_acquire 1736 #endif 1737 1738 #if defined(AO_HAVE_int_fetch_compare_and_swap_release) \ 1739 && !defined(AO_HAVE_int_compare_and_swap_release) 1740 AO_INLINE int AO_int_compare_and_swap_release(volatile unsigned * addr,unsigned old_val,unsigned new_val)1741 AO_int_compare_and_swap_release(volatile unsigned *addr, unsigned old_val, 1742 unsigned new_val) 1743 { 1744 return AO_int_fetch_compare_and_swap_release(addr, old_val, new_val) 1745 == old_val; 1746 } 1747 # define AO_HAVE_int_compare_and_swap_release 1748 #endif 1749 1750 #if defined(AO_HAVE_int_fetch_compare_and_swap_write) \ 1751 && !defined(AO_HAVE_int_compare_and_swap_write) 1752 AO_INLINE int AO_int_compare_and_swap_write(volatile unsigned * addr,unsigned old_val,unsigned new_val)1753 AO_int_compare_and_swap_write(volatile unsigned *addr, unsigned old_val, 1754 unsigned new_val) 1755 { 1756 return AO_int_fetch_compare_and_swap_write(addr, old_val, new_val) 1757 == old_val; 1758 } 1759 # define AO_HAVE_int_compare_and_swap_write 1760 #endif 1761 1762 #if defined(AO_HAVE_int_fetch_compare_and_swap_read) \ 1763 && !defined(AO_HAVE_int_compare_and_swap_read) 1764 AO_INLINE int AO_int_compare_and_swap_read(volatile unsigned * addr,unsigned old_val,unsigned new_val)1765 AO_int_compare_and_swap_read(volatile unsigned *addr, unsigned old_val, 1766 unsigned new_val) 1767 { 1768 return AO_int_fetch_compare_and_swap_read(addr, old_val, new_val) 1769 == old_val; 1770 } 1771 # define AO_HAVE_int_compare_and_swap_read 1772 #endif 1773 1774 #if defined(AO_HAVE_int_fetch_compare_and_swap) \ 1775 && !defined(AO_HAVE_int_compare_and_swap) 1776 AO_INLINE int AO_int_compare_and_swap(volatile unsigned * addr,unsigned old_val,unsigned new_val)1777 AO_int_compare_and_swap(volatile unsigned *addr, unsigned old_val, 1778 unsigned new_val) 1779 { 1780 return AO_int_fetch_compare_and_swap(addr, old_val, new_val) == old_val; 1781 } 1782 # define AO_HAVE_int_compare_and_swap 1783 #endif 1784 1785 #if defined(AO_HAVE_int_fetch_compare_and_swap_release_write) \ 1786 && !defined(AO_HAVE_int_compare_and_swap_release_write) 1787 AO_INLINE int AO_int_compare_and_swap_release_write(volatile unsigned * addr,unsigned old_val,unsigned new_val)1788 AO_int_compare_and_swap_release_write(volatile unsigned *addr, 1789 unsigned old_val, unsigned new_val) 1790 { 1791 return AO_int_fetch_compare_and_swap_release_write(addr, old_val, 1792 new_val) == old_val; 1793 } 1794 # define AO_HAVE_int_compare_and_swap_release_write 1795 #endif 1796 1797 #if defined(AO_HAVE_int_fetch_compare_and_swap_acquire_read) \ 1798 && !defined(AO_HAVE_int_compare_and_swap_acquire_read) 1799 AO_INLINE int AO_int_compare_and_swap_acquire_read(volatile unsigned * addr,unsigned old_val,unsigned new_val)1800 AO_int_compare_and_swap_acquire_read(volatile unsigned *addr, 1801 unsigned old_val, unsigned new_val) 1802 { 1803 return AO_int_fetch_compare_and_swap_acquire_read(addr, old_val, 1804 new_val) == old_val; 1805 } 1806 # define AO_HAVE_int_compare_and_swap_acquire_read 1807 #endif 1808 1809 #if defined(AO_HAVE_int_fetch_compare_and_swap_dd_acquire_read) \ 1810 && !defined(AO_HAVE_int_compare_and_swap_dd_acquire_read) 1811 AO_INLINE int AO_int_compare_and_swap_dd_acquire_read(volatile unsigned * addr,unsigned old_val,unsigned new_val)1812 AO_int_compare_and_swap_dd_acquire_read(volatile unsigned *addr, 1813 unsigned old_val, unsigned new_val) 1814 { 1815 return AO_int_fetch_compare_and_swap_dd_acquire_read(addr, old_val, 1816 new_val) == old_val; 1817 } 1818 # define AO_HAVE_int_compare_and_swap_dd_acquire_read 1819 #endif 1820 1821 /* int_fetch_and_add */ 1822 /* We first try to implement fetch_and_add variants in terms of the */ 1823 /* corresponding compare_and_swap variants to minimize adding barriers. */ 1824 #if defined(AO_HAVE_int_compare_and_swap_full) \ 1825 && !defined(AO_HAVE_int_fetch_and_add_full) 1826 AO_INLINE unsigned AO_int_fetch_and_add_full(volatile unsigned * addr,unsigned incr)1827 AO_int_fetch_and_add_full(volatile unsigned *addr, unsigned incr) 1828 { 1829 unsigned old; 1830 1831 do 1832 { 1833 old = *(unsigned *)addr; 1834 } 1835 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, 1836 old + incr))); 1837 return old; 1838 } 1839 # define AO_HAVE_int_fetch_and_add_full 1840 #endif 1841 1842 #if defined(AO_HAVE_int_compare_and_swap_acquire) \ 1843 && !defined(AO_HAVE_int_fetch_and_add_acquire) 1844 AO_INLINE unsigned AO_int_fetch_and_add_acquire(volatile unsigned * addr,unsigned incr)1845 AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr) 1846 { 1847 unsigned old; 1848 1849 do 1850 { 1851 old = *(unsigned *)addr; 1852 } 1853 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_acquire(addr, old, 1854 old + incr))); 1855 return old; 1856 } 1857 # define AO_HAVE_int_fetch_and_add_acquire 1858 #endif 1859 1860 #if defined(AO_HAVE_int_compare_and_swap_release) \ 1861 && !defined(AO_HAVE_int_fetch_and_add_release) 1862 AO_INLINE unsigned AO_int_fetch_and_add_release(volatile unsigned * addr,unsigned incr)1863 AO_int_fetch_and_add_release(volatile unsigned *addr, unsigned incr) 1864 { 1865 unsigned old; 1866 1867 do 1868 { 1869 old = *(unsigned *)addr; 1870 } 1871 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_release(addr, old, 1872 old + incr))); 1873 return old; 1874 } 1875 # define AO_HAVE_int_fetch_and_add_release 1876 #endif 1877 1878 #if defined(AO_HAVE_int_compare_and_swap) \ 1879 && !defined(AO_HAVE_int_fetch_and_add) 1880 AO_INLINE unsigned AO_int_fetch_and_add(volatile unsigned * addr,unsigned incr)1881 AO_int_fetch_and_add(volatile unsigned *addr, unsigned incr) 1882 { 1883 unsigned old; 1884 1885 do 1886 { 1887 old = *(unsigned *)addr; 1888 } 1889 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap(addr, old, 1890 old + incr))); 1891 return old; 1892 } 1893 # define AO_HAVE_int_fetch_and_add 1894 #endif 1895 1896 #if defined(AO_HAVE_int_fetch_and_add_full) 1897 # if !defined(AO_HAVE_int_fetch_and_add_release) 1898 # define AO_int_fetch_and_add_release(addr, val) \ 1899 AO_int_fetch_and_add_full(addr, val) 1900 # define AO_HAVE_int_fetch_and_add_release 1901 # endif 1902 # if !defined(AO_HAVE_int_fetch_and_add_acquire) 1903 # define AO_int_fetch_and_add_acquire(addr, val) \ 1904 AO_int_fetch_and_add_full(addr, val) 1905 # define AO_HAVE_int_fetch_and_add_acquire 1906 # endif 1907 # if !defined(AO_HAVE_int_fetch_and_add_write) 1908 # define AO_int_fetch_and_add_write(addr, val) \ 1909 AO_int_fetch_and_add_full(addr, val) 1910 # define AO_HAVE_int_fetch_and_add_write 1911 # endif 1912 # if !defined(AO_HAVE_int_fetch_and_add_read) 1913 # define AO_int_fetch_and_add_read(addr, val) \ 1914 AO_int_fetch_and_add_full(addr, val) 1915 # define AO_HAVE_int_fetch_and_add_read 1916 # endif 1917 #endif /* AO_HAVE_int_fetch_and_add_full */ 1918 1919 #if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \ 1920 && !defined(AO_HAVE_int_fetch_and_add_acquire) 1921 AO_INLINE unsigned AO_int_fetch_and_add_acquire(volatile unsigned * addr,unsigned incr)1922 AO_int_fetch_and_add_acquire(volatile unsigned *addr, unsigned incr) 1923 { 1924 unsigned result = AO_int_fetch_and_add(addr, incr); 1925 AO_nop_full(); 1926 return result; 1927 } 1928 # define AO_HAVE_int_fetch_and_add_acquire 1929 #endif 1930 #if defined(AO_HAVE_int_fetch_and_add) && defined(AO_HAVE_nop_full) \ 1931 && !defined(AO_HAVE_int_fetch_and_add_release) 1932 # define AO_int_fetch_and_add_release(addr, incr) \ 1933 (AO_nop_full(), AO_int_fetch_and_add(addr, incr)) 1934 # define AO_HAVE_int_fetch_and_add_release 1935 #endif 1936 1937 #if !defined(AO_HAVE_int_fetch_and_add) \ 1938 && defined(AO_HAVE_int_fetch_and_add_release) 1939 # define AO_int_fetch_and_add(addr, val) \ 1940 AO_int_fetch_and_add_release(addr, val) 1941 # define AO_HAVE_int_fetch_and_add 1942 #endif 1943 #if !defined(AO_HAVE_int_fetch_and_add) \ 1944 && defined(AO_HAVE_int_fetch_and_add_acquire) 1945 # define AO_int_fetch_and_add(addr, val) \ 1946 AO_int_fetch_and_add_acquire(addr, val) 1947 # define AO_HAVE_int_fetch_and_add 1948 #endif 1949 #if !defined(AO_HAVE_int_fetch_and_add) \ 1950 && defined(AO_HAVE_int_fetch_and_add_write) 1951 # define AO_int_fetch_and_add(addr, val) \ 1952 AO_int_fetch_and_add_write(addr, val) 1953 # define AO_HAVE_int_fetch_and_add 1954 #endif 1955 #if !defined(AO_HAVE_int_fetch_and_add) \ 1956 && defined(AO_HAVE_int_fetch_and_add_read) 1957 # define AO_int_fetch_and_add(addr, val) \ 1958 AO_int_fetch_and_add_read(addr, val) 1959 # define AO_HAVE_int_fetch_and_add 1960 #endif 1961 1962 #if defined(AO_HAVE_int_fetch_and_add_acquire) \ 1963 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_int_fetch_and_add_full) 1964 # define AO_int_fetch_and_add_full(addr, val) \ 1965 (AO_nop_full(), AO_int_fetch_and_add_acquire(addr, val)) 1966 # define AO_HAVE_int_fetch_and_add_full 1967 #endif 1968 1969 #if !defined(AO_HAVE_int_fetch_and_add_release_write) \ 1970 && defined(AO_HAVE_int_fetch_and_add_write) 1971 # define AO_int_fetch_and_add_release_write(addr, val) \ 1972 AO_int_fetch_and_add_write(addr, val) 1973 # define AO_HAVE_int_fetch_and_add_release_write 1974 #endif 1975 #if !defined(AO_HAVE_int_fetch_and_add_release_write) \ 1976 && defined(AO_HAVE_int_fetch_and_add_release) 1977 # define AO_int_fetch_and_add_release_write(addr, val) \ 1978 AO_int_fetch_and_add_release(addr, val) 1979 # define AO_HAVE_int_fetch_and_add_release_write 1980 #endif 1981 1982 #if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \ 1983 && defined(AO_HAVE_int_fetch_and_add_read) 1984 # define AO_int_fetch_and_add_acquire_read(addr, val) \ 1985 AO_int_fetch_and_add_read(addr, val) 1986 # define AO_HAVE_int_fetch_and_add_acquire_read 1987 #endif 1988 #if !defined(AO_HAVE_int_fetch_and_add_acquire_read) \ 1989 && defined(AO_HAVE_int_fetch_and_add_acquire) 1990 # define AO_int_fetch_and_add_acquire_read(addr, val) \ 1991 AO_int_fetch_and_add_acquire(addr, val) 1992 # define AO_HAVE_int_fetch_and_add_acquire_read 1993 #endif 1994 1995 #ifdef AO_NO_DD_ORDERING 1996 # if defined(AO_HAVE_int_fetch_and_add_acquire_read) 1997 # define AO_int_fetch_and_add_dd_acquire_read(addr, val) \ 1998 AO_int_fetch_and_add_acquire_read(addr, val) 1999 # define AO_HAVE_int_fetch_and_add_dd_acquire_read 2000 # endif 2001 #else 2002 # if defined(AO_HAVE_int_fetch_and_add) 2003 # define AO_int_fetch_and_add_dd_acquire_read(addr, val) \ 2004 AO_int_fetch_and_add(addr, val) 2005 # define AO_HAVE_int_fetch_and_add_dd_acquire_read 2006 # endif 2007 #endif /* !AO_NO_DD_ORDERING */ 2008 2009 /* int_fetch_and_add1 */ 2010 #if defined(AO_HAVE_int_fetch_and_add_full) \ 2011 && !defined(AO_HAVE_int_fetch_and_add1_full) 2012 # define AO_int_fetch_and_add1_full(addr) \ 2013 AO_int_fetch_and_add_full(addr, 1) 2014 # define AO_HAVE_int_fetch_and_add1_full 2015 #endif 2016 #if defined(AO_HAVE_int_fetch_and_add_release) \ 2017 && !defined(AO_HAVE_int_fetch_and_add1_release) 2018 # define AO_int_fetch_and_add1_release(addr) \ 2019 AO_int_fetch_and_add_release(addr, 1) 2020 # define AO_HAVE_int_fetch_and_add1_release 2021 #endif 2022 #if defined(AO_HAVE_int_fetch_and_add_acquire) \ 2023 && !defined(AO_HAVE_int_fetch_and_add1_acquire) 2024 # define AO_int_fetch_and_add1_acquire(addr) \ 2025 AO_int_fetch_and_add_acquire(addr, 1) 2026 # define AO_HAVE_int_fetch_and_add1_acquire 2027 #endif 2028 #if defined(AO_HAVE_int_fetch_and_add_write) \ 2029 && !defined(AO_HAVE_int_fetch_and_add1_write) 2030 # define AO_int_fetch_and_add1_write(addr) \ 2031 AO_int_fetch_and_add_write(addr, 1) 2032 # define AO_HAVE_int_fetch_and_add1_write 2033 #endif 2034 #if defined(AO_HAVE_int_fetch_and_add_read) \ 2035 && !defined(AO_HAVE_int_fetch_and_add1_read) 2036 # define AO_int_fetch_and_add1_read(addr) \ 2037 AO_int_fetch_and_add_read(addr, 1) 2038 # define AO_HAVE_int_fetch_and_add1_read 2039 #endif 2040 #if defined(AO_HAVE_int_fetch_and_add_release_write) \ 2041 && !defined(AO_HAVE_int_fetch_and_add1_release_write) 2042 # define AO_int_fetch_and_add1_release_write(addr) \ 2043 AO_int_fetch_and_add_release_write(addr, 1) 2044 # define AO_HAVE_int_fetch_and_add1_release_write 2045 #endif 2046 #if defined(AO_HAVE_int_fetch_and_add_acquire_read) \ 2047 && !defined(AO_HAVE_int_fetch_and_add1_acquire_read) 2048 # define AO_int_fetch_and_add1_acquire_read(addr) \ 2049 AO_int_fetch_and_add_acquire_read(addr, 1) 2050 # define AO_HAVE_int_fetch_and_add1_acquire_read 2051 #endif 2052 #if defined(AO_HAVE_int_fetch_and_add) \ 2053 && !defined(AO_HAVE_int_fetch_and_add1) 2054 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add(addr, 1) 2055 # define AO_HAVE_int_fetch_and_add1 2056 #endif 2057 2058 #if defined(AO_HAVE_int_fetch_and_add1_full) 2059 # if !defined(AO_HAVE_int_fetch_and_add1_release) 2060 # define AO_int_fetch_and_add1_release(addr) \ 2061 AO_int_fetch_and_add1_full(addr) 2062 # define AO_HAVE_int_fetch_and_add1_release 2063 # endif 2064 # if !defined(AO_HAVE_int_fetch_and_add1_acquire) 2065 # define AO_int_fetch_and_add1_acquire(addr) \ 2066 AO_int_fetch_and_add1_full(addr) 2067 # define AO_HAVE_int_fetch_and_add1_acquire 2068 # endif 2069 # if !defined(AO_HAVE_int_fetch_and_add1_write) 2070 # define AO_int_fetch_and_add1_write(addr) \ 2071 AO_int_fetch_and_add1_full(addr) 2072 # define AO_HAVE_int_fetch_and_add1_write 2073 # endif 2074 # if !defined(AO_HAVE_int_fetch_and_add1_read) 2075 # define AO_int_fetch_and_add1_read(addr) \ 2076 AO_int_fetch_and_add1_full(addr) 2077 # define AO_HAVE_int_fetch_and_add1_read 2078 # endif 2079 #endif /* AO_HAVE_int_fetch_and_add1_full */ 2080 2081 #if !defined(AO_HAVE_int_fetch_and_add1) \ 2082 && defined(AO_HAVE_int_fetch_and_add1_release) 2083 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_release(addr) 2084 # define AO_HAVE_int_fetch_and_add1 2085 #endif 2086 #if !defined(AO_HAVE_int_fetch_and_add1) \ 2087 && defined(AO_HAVE_int_fetch_and_add1_acquire) 2088 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_acquire(addr) 2089 # define AO_HAVE_int_fetch_and_add1 2090 #endif 2091 #if !defined(AO_HAVE_int_fetch_and_add1) \ 2092 && defined(AO_HAVE_int_fetch_and_add1_write) 2093 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_write(addr) 2094 # define AO_HAVE_int_fetch_and_add1 2095 #endif 2096 #if !defined(AO_HAVE_int_fetch_and_add1) \ 2097 && defined(AO_HAVE_int_fetch_and_add1_read) 2098 # define AO_int_fetch_and_add1(addr) AO_int_fetch_and_add1_read(addr) 2099 # define AO_HAVE_int_fetch_and_add1 2100 #endif 2101 2102 #if defined(AO_HAVE_int_fetch_and_add1_acquire) \ 2103 && defined(AO_HAVE_nop_full) \ 2104 && !defined(AO_HAVE_int_fetch_and_add1_full) 2105 # define AO_int_fetch_and_add1_full(addr) \ 2106 (AO_nop_full(), AO_int_fetch_and_add1_acquire(addr)) 2107 # define AO_HAVE_int_fetch_and_add1_full 2108 #endif 2109 2110 #if !defined(AO_HAVE_int_fetch_and_add1_release_write) \ 2111 && defined(AO_HAVE_int_fetch_and_add1_write) 2112 # define AO_int_fetch_and_add1_release_write(addr) \ 2113 AO_int_fetch_and_add1_write(addr) 2114 # define AO_HAVE_int_fetch_and_add1_release_write 2115 #endif 2116 #if !defined(AO_HAVE_int_fetch_and_add1_release_write) \ 2117 && defined(AO_HAVE_int_fetch_and_add1_release) 2118 # define AO_int_fetch_and_add1_release_write(addr) \ 2119 AO_int_fetch_and_add1_release(addr) 2120 # define AO_HAVE_int_fetch_and_add1_release_write 2121 #endif 2122 #if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \ 2123 && defined(AO_HAVE_int_fetch_and_add1_read) 2124 # define AO_int_fetch_and_add1_acquire_read(addr) \ 2125 AO_int_fetch_and_add1_read(addr) 2126 # define AO_HAVE_int_fetch_and_add1_acquire_read 2127 #endif 2128 #if !defined(AO_HAVE_int_fetch_and_add1_acquire_read) \ 2129 && defined(AO_HAVE_int_fetch_and_add1_acquire) 2130 # define AO_int_fetch_and_add1_acquire_read(addr) \ 2131 AO_int_fetch_and_add1_acquire(addr) 2132 # define AO_HAVE_int_fetch_and_add1_acquire_read 2133 #endif 2134 2135 #ifdef AO_NO_DD_ORDERING 2136 # if defined(AO_HAVE_int_fetch_and_add1_acquire_read) 2137 # define AO_int_fetch_and_add1_dd_acquire_read(addr) \ 2138 AO_int_fetch_and_add1_acquire_read(addr) 2139 # define AO_HAVE_int_fetch_and_add1_dd_acquire_read 2140 # endif 2141 #else 2142 # if defined(AO_HAVE_int_fetch_and_add1) 2143 # define AO_int_fetch_and_add1_dd_acquire_read(addr) \ 2144 AO_int_fetch_and_add1(addr) 2145 # define AO_HAVE_int_fetch_and_add1_dd_acquire_read 2146 # endif 2147 #endif /* !AO_NO_DD_ORDERING */ 2148 2149 /* int_fetch_and_sub1 */ 2150 #if defined(AO_HAVE_int_fetch_and_add_full) \ 2151 && !defined(AO_HAVE_int_fetch_and_sub1_full) 2152 # define AO_int_fetch_and_sub1_full(addr) \ 2153 AO_int_fetch_and_add_full(addr, (unsigned)(-1)) 2154 # define AO_HAVE_int_fetch_and_sub1_full 2155 #endif 2156 #if defined(AO_HAVE_int_fetch_and_add_release) \ 2157 && !defined(AO_HAVE_int_fetch_and_sub1_release) 2158 # define AO_int_fetch_and_sub1_release(addr) \ 2159 AO_int_fetch_and_add_release(addr, (unsigned)(-1)) 2160 # define AO_HAVE_int_fetch_and_sub1_release 2161 #endif 2162 #if defined(AO_HAVE_int_fetch_and_add_acquire) \ 2163 && !defined(AO_HAVE_int_fetch_and_sub1_acquire) 2164 # define AO_int_fetch_and_sub1_acquire(addr) \ 2165 AO_int_fetch_and_add_acquire(addr, (unsigned)(-1)) 2166 # define AO_HAVE_int_fetch_and_sub1_acquire 2167 #endif 2168 #if defined(AO_HAVE_int_fetch_and_add_write) \ 2169 && !defined(AO_HAVE_int_fetch_and_sub1_write) 2170 # define AO_int_fetch_and_sub1_write(addr) \ 2171 AO_int_fetch_and_add_write(addr, (unsigned)(-1)) 2172 # define AO_HAVE_int_fetch_and_sub1_write 2173 #endif 2174 #if defined(AO_HAVE_int_fetch_and_add_read) \ 2175 && !defined(AO_HAVE_int_fetch_and_sub1_read) 2176 # define AO_int_fetch_and_sub1_read(addr) \ 2177 AO_int_fetch_and_add_read(addr, (unsigned)(-1)) 2178 # define AO_HAVE_int_fetch_and_sub1_read 2179 #endif 2180 #if defined(AO_HAVE_int_fetch_and_add_release_write) \ 2181 && !defined(AO_HAVE_int_fetch_and_sub1_release_write) 2182 # define AO_int_fetch_and_sub1_release_write(addr) \ 2183 AO_int_fetch_and_add_release_write(addr, (unsigned)(-1)) 2184 # define AO_HAVE_int_fetch_and_sub1_release_write 2185 #endif 2186 #if defined(AO_HAVE_int_fetch_and_add_acquire_read) \ 2187 && !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) 2188 # define AO_int_fetch_and_sub1_acquire_read(addr) \ 2189 AO_int_fetch_and_add_acquire_read(addr, (unsigned)(-1)) 2190 # define AO_HAVE_int_fetch_and_sub1_acquire_read 2191 #endif 2192 #if defined(AO_HAVE_int_fetch_and_add) \ 2193 && !defined(AO_HAVE_int_fetch_and_sub1) 2194 # define AO_int_fetch_and_sub1(addr) \ 2195 AO_int_fetch_and_add(addr, (unsigned)(-1)) 2196 # define AO_HAVE_int_fetch_and_sub1 2197 #endif 2198 2199 #if defined(AO_HAVE_int_fetch_and_sub1_full) 2200 # if !defined(AO_HAVE_int_fetch_and_sub1_release) 2201 # define AO_int_fetch_and_sub1_release(addr) \ 2202 AO_int_fetch_and_sub1_full(addr) 2203 # define AO_HAVE_int_fetch_and_sub1_release 2204 # endif 2205 # if !defined(AO_HAVE_int_fetch_and_sub1_acquire) 2206 # define AO_int_fetch_and_sub1_acquire(addr) \ 2207 AO_int_fetch_and_sub1_full(addr) 2208 # define AO_HAVE_int_fetch_and_sub1_acquire 2209 # endif 2210 # if !defined(AO_HAVE_int_fetch_and_sub1_write) 2211 # define AO_int_fetch_and_sub1_write(addr) \ 2212 AO_int_fetch_and_sub1_full(addr) 2213 # define AO_HAVE_int_fetch_and_sub1_write 2214 # endif 2215 # if !defined(AO_HAVE_int_fetch_and_sub1_read) 2216 # define AO_int_fetch_and_sub1_read(addr) \ 2217 AO_int_fetch_and_sub1_full(addr) 2218 # define AO_HAVE_int_fetch_and_sub1_read 2219 # endif 2220 #endif /* AO_HAVE_int_fetch_and_sub1_full */ 2221 2222 #if !defined(AO_HAVE_int_fetch_and_sub1) \ 2223 && defined(AO_HAVE_int_fetch_and_sub1_release) 2224 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_release(addr) 2225 # define AO_HAVE_int_fetch_and_sub1 2226 #endif 2227 #if !defined(AO_HAVE_int_fetch_and_sub1) \ 2228 && defined(AO_HAVE_int_fetch_and_sub1_acquire) 2229 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_acquire(addr) 2230 # define AO_HAVE_int_fetch_and_sub1 2231 #endif 2232 #if !defined(AO_HAVE_int_fetch_and_sub1) \ 2233 && defined(AO_HAVE_int_fetch_and_sub1_write) 2234 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_write(addr) 2235 # define AO_HAVE_int_fetch_and_sub1 2236 #endif 2237 #if !defined(AO_HAVE_int_fetch_and_sub1) \ 2238 && defined(AO_HAVE_int_fetch_and_sub1_read) 2239 # define AO_int_fetch_and_sub1(addr) AO_int_fetch_and_sub1_read(addr) 2240 # define AO_HAVE_int_fetch_and_sub1 2241 #endif 2242 2243 #if defined(AO_HAVE_int_fetch_and_sub1_acquire) \ 2244 && defined(AO_HAVE_nop_full) \ 2245 && !defined(AO_HAVE_int_fetch_and_sub1_full) 2246 # define AO_int_fetch_and_sub1_full(addr) \ 2247 (AO_nop_full(), AO_int_fetch_and_sub1_acquire(addr)) 2248 # define AO_HAVE_int_fetch_and_sub1_full 2249 #endif 2250 2251 #if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \ 2252 && defined(AO_HAVE_int_fetch_and_sub1_write) 2253 # define AO_int_fetch_and_sub1_release_write(addr) \ 2254 AO_int_fetch_and_sub1_write(addr) 2255 # define AO_HAVE_int_fetch_and_sub1_release_write 2256 #endif 2257 #if !defined(AO_HAVE_int_fetch_and_sub1_release_write) \ 2258 && defined(AO_HAVE_int_fetch_and_sub1_release) 2259 # define AO_int_fetch_and_sub1_release_write(addr) \ 2260 AO_int_fetch_and_sub1_release(addr) 2261 # define AO_HAVE_int_fetch_and_sub1_release_write 2262 #endif 2263 #if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \ 2264 && defined(AO_HAVE_int_fetch_and_sub1_read) 2265 # define AO_int_fetch_and_sub1_acquire_read(addr) \ 2266 AO_int_fetch_and_sub1_read(addr) 2267 # define AO_HAVE_int_fetch_and_sub1_acquire_read 2268 #endif 2269 #if !defined(AO_HAVE_int_fetch_and_sub1_acquire_read) \ 2270 && defined(AO_HAVE_int_fetch_and_sub1_acquire) 2271 # define AO_int_fetch_and_sub1_acquire_read(addr) \ 2272 AO_int_fetch_and_sub1_acquire(addr) 2273 # define AO_HAVE_int_fetch_and_sub1_acquire_read 2274 #endif 2275 2276 #ifdef AO_NO_DD_ORDERING 2277 # if defined(AO_HAVE_int_fetch_and_sub1_acquire_read) 2278 # define AO_int_fetch_and_sub1_dd_acquire_read(addr) \ 2279 AO_int_fetch_and_sub1_acquire_read(addr) 2280 # define AO_HAVE_int_fetch_and_sub1_dd_acquire_read 2281 # endif 2282 #else 2283 # if defined(AO_HAVE_int_fetch_and_sub1) 2284 # define AO_int_fetch_and_sub1_dd_acquire_read(addr) \ 2285 AO_int_fetch_and_sub1(addr) 2286 # define AO_HAVE_int_fetch_and_sub1_dd_acquire_read 2287 # endif 2288 #endif /* !AO_NO_DD_ORDERING */ 2289 2290 /* int_and */ 2291 #if defined(AO_HAVE_int_compare_and_swap_full) \ 2292 && !defined(AO_HAVE_int_and_full) 2293 AO_INLINE void AO_int_and_full(volatile unsigned * addr,unsigned value)2294 AO_int_and_full(volatile unsigned *addr, unsigned value) 2295 { 2296 unsigned old; 2297 2298 do 2299 { 2300 old = *(unsigned *)addr; 2301 } 2302 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, 2303 old & value))); 2304 } 2305 # define AO_HAVE_int_and_full 2306 #endif 2307 2308 #if defined(AO_HAVE_int_and_full) 2309 # if !defined(AO_HAVE_int_and_release) 2310 # define AO_int_and_release(addr, val) AO_int_and_full(addr, val) 2311 # define AO_HAVE_int_and_release 2312 # endif 2313 # if !defined(AO_HAVE_int_and_acquire) 2314 # define AO_int_and_acquire(addr, val) AO_int_and_full(addr, val) 2315 # define AO_HAVE_int_and_acquire 2316 # endif 2317 # if !defined(AO_HAVE_int_and_write) 2318 # define AO_int_and_write(addr, val) AO_int_and_full(addr, val) 2319 # define AO_HAVE_int_and_write 2320 # endif 2321 # if !defined(AO_HAVE_int_and_read) 2322 # define AO_int_and_read(addr, val) AO_int_and_full(addr, val) 2323 # define AO_HAVE_int_and_read 2324 # endif 2325 #endif /* AO_HAVE_int_and_full */ 2326 2327 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_release) 2328 # define AO_int_and(addr, val) AO_int_and_release(addr, val) 2329 # define AO_HAVE_int_and 2330 #endif 2331 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_acquire) 2332 # define AO_int_and(addr, val) AO_int_and_acquire(addr, val) 2333 # define AO_HAVE_int_and 2334 #endif 2335 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_write) 2336 # define AO_int_and(addr, val) AO_int_and_write(addr, val) 2337 # define AO_HAVE_int_and 2338 #endif 2339 #if !defined(AO_HAVE_int_and) && defined(AO_HAVE_int_and_read) 2340 # define AO_int_and(addr, val) AO_int_and_read(addr, val) 2341 # define AO_HAVE_int_and 2342 #endif 2343 2344 #if defined(AO_HAVE_int_and_acquire) && defined(AO_HAVE_nop_full) \ 2345 && !defined(AO_HAVE_int_and_full) 2346 # define AO_int_and_full(addr, val) \ 2347 (AO_nop_full(), AO_int_and_acquire(addr, val)) 2348 # define AO_HAVE_int_and_full 2349 #endif 2350 2351 #if !defined(AO_HAVE_int_and_release_write) \ 2352 && defined(AO_HAVE_int_and_write) 2353 # define AO_int_and_release_write(addr, val) AO_int_and_write(addr, val) 2354 # define AO_HAVE_int_and_release_write 2355 #endif 2356 #if !defined(AO_HAVE_int_and_release_write) \ 2357 && defined(AO_HAVE_int_and_release) 2358 # define AO_int_and_release_write(addr, val) AO_int_and_release(addr, val) 2359 # define AO_HAVE_int_and_release_write 2360 #endif 2361 #if !defined(AO_HAVE_int_and_acquire_read) \ 2362 && defined(AO_HAVE_int_and_read) 2363 # define AO_int_and_acquire_read(addr, val) AO_int_and_read(addr, val) 2364 # define AO_HAVE_int_and_acquire_read 2365 #endif 2366 #if !defined(AO_HAVE_int_and_acquire_read) \ 2367 && defined(AO_HAVE_int_and_acquire) 2368 # define AO_int_and_acquire_read(addr, val) AO_int_and_acquire(addr, val) 2369 # define AO_HAVE_int_and_acquire_read 2370 #endif 2371 2372 /* int_or */ 2373 #if defined(AO_HAVE_int_compare_and_swap_full) \ 2374 && !defined(AO_HAVE_int_or_full) 2375 AO_INLINE void AO_int_or_full(volatile unsigned * addr,unsigned value)2376 AO_int_or_full(volatile unsigned *addr, unsigned value) 2377 { 2378 unsigned old; 2379 2380 do 2381 { 2382 old = *(unsigned *)addr; 2383 } 2384 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, 2385 old | value))); 2386 } 2387 # define AO_HAVE_int_or_full 2388 #endif 2389 2390 #if defined(AO_HAVE_int_or_full) 2391 # if !defined(AO_HAVE_int_or_release) 2392 # define AO_int_or_release(addr, val) AO_int_or_full(addr, val) 2393 # define AO_HAVE_int_or_release 2394 # endif 2395 # if !defined(AO_HAVE_int_or_acquire) 2396 # define AO_int_or_acquire(addr, val) AO_int_or_full(addr, val) 2397 # define AO_HAVE_int_or_acquire 2398 # endif 2399 # if !defined(AO_HAVE_int_or_write) 2400 # define AO_int_or_write(addr, val) AO_int_or_full(addr, val) 2401 # define AO_HAVE_int_or_write 2402 # endif 2403 # if !defined(AO_HAVE_int_or_read) 2404 # define AO_int_or_read(addr, val) AO_int_or_full(addr, val) 2405 # define AO_HAVE_int_or_read 2406 # endif 2407 #endif /* AO_HAVE_int_or_full */ 2408 2409 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_release) 2410 # define AO_int_or(addr, val) AO_int_or_release(addr, val) 2411 # define AO_HAVE_int_or 2412 #endif 2413 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_acquire) 2414 # define AO_int_or(addr, val) AO_int_or_acquire(addr, val) 2415 # define AO_HAVE_int_or 2416 #endif 2417 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_write) 2418 # define AO_int_or(addr, val) AO_int_or_write(addr, val) 2419 # define AO_HAVE_int_or 2420 #endif 2421 #if !defined(AO_HAVE_int_or) && defined(AO_HAVE_int_or_read) 2422 # define AO_int_or(addr, val) AO_int_or_read(addr, val) 2423 # define AO_HAVE_int_or 2424 #endif 2425 2426 #if defined(AO_HAVE_int_or_acquire) && defined(AO_HAVE_nop_full) \ 2427 && !defined(AO_HAVE_int_or_full) 2428 # define AO_int_or_full(addr, val) \ 2429 (AO_nop_full(), AO_int_or_acquire(addr, val)) 2430 # define AO_HAVE_int_or_full 2431 #endif 2432 2433 #if !defined(AO_HAVE_int_or_release_write) \ 2434 && defined(AO_HAVE_int_or_write) 2435 # define AO_int_or_release_write(addr, val) AO_int_or_write(addr, val) 2436 # define AO_HAVE_int_or_release_write 2437 #endif 2438 #if !defined(AO_HAVE_int_or_release_write) \ 2439 && defined(AO_HAVE_int_or_release) 2440 # define AO_int_or_release_write(addr, val) AO_int_or_release(addr, val) 2441 # define AO_HAVE_int_or_release_write 2442 #endif 2443 #if !defined(AO_HAVE_int_or_acquire_read) && defined(AO_HAVE_int_or_read) 2444 # define AO_int_or_acquire_read(addr, val) AO_int_or_read(addr, val) 2445 # define AO_HAVE_int_or_acquire_read 2446 #endif 2447 #if !defined(AO_HAVE_int_or_acquire_read) \ 2448 && defined(AO_HAVE_int_or_acquire) 2449 # define AO_int_or_acquire_read(addr, val) AO_int_or_acquire(addr, val) 2450 # define AO_HAVE_int_or_acquire_read 2451 #endif 2452 2453 /* int_xor */ 2454 #if defined(AO_HAVE_int_compare_and_swap_full) \ 2455 && !defined(AO_HAVE_int_xor_full) 2456 AO_INLINE void AO_int_xor_full(volatile unsigned * addr,unsigned value)2457 AO_int_xor_full(volatile unsigned *addr, unsigned value) 2458 { 2459 unsigned old; 2460 2461 do 2462 { 2463 old = *(unsigned *)addr; 2464 } 2465 while (AO_EXPECT_FALSE(!AO_int_compare_and_swap_full(addr, old, 2466 old ^ value))); 2467 } 2468 # define AO_HAVE_int_xor_full 2469 #endif 2470 2471 #if defined(AO_HAVE_int_xor_full) 2472 # if !defined(AO_HAVE_int_xor_release) 2473 # define AO_int_xor_release(addr, val) AO_int_xor_full(addr, val) 2474 # define AO_HAVE_int_xor_release 2475 # endif 2476 # if !defined(AO_HAVE_int_xor_acquire) 2477 # define AO_int_xor_acquire(addr, val) AO_int_xor_full(addr, val) 2478 # define AO_HAVE_int_xor_acquire 2479 # endif 2480 # if !defined(AO_HAVE_int_xor_write) 2481 # define AO_int_xor_write(addr, val) AO_int_xor_full(addr, val) 2482 # define AO_HAVE_int_xor_write 2483 # endif 2484 # if !defined(AO_HAVE_int_xor_read) 2485 # define AO_int_xor_read(addr, val) AO_int_xor_full(addr, val) 2486 # define AO_HAVE_int_xor_read 2487 # endif 2488 #endif /* AO_HAVE_int_xor_full */ 2489 2490 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_release) 2491 # define AO_int_xor(addr, val) AO_int_xor_release(addr, val) 2492 # define AO_HAVE_int_xor 2493 #endif 2494 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_acquire) 2495 # define AO_int_xor(addr, val) AO_int_xor_acquire(addr, val) 2496 # define AO_HAVE_int_xor 2497 #endif 2498 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_write) 2499 # define AO_int_xor(addr, val) AO_int_xor_write(addr, val) 2500 # define AO_HAVE_int_xor 2501 #endif 2502 #if !defined(AO_HAVE_int_xor) && defined(AO_HAVE_int_xor_read) 2503 # define AO_int_xor(addr, val) AO_int_xor_read(addr, val) 2504 # define AO_HAVE_int_xor 2505 #endif 2506 2507 #if defined(AO_HAVE_int_xor_acquire) && defined(AO_HAVE_nop_full) \ 2508 && !defined(AO_HAVE_int_xor_full) 2509 # define AO_int_xor_full(addr, val) \ 2510 (AO_nop_full(), AO_int_xor_acquire(addr, val)) 2511 # define AO_HAVE_int_xor_full 2512 #endif 2513 2514 #if !defined(AO_HAVE_int_xor_release_write) \ 2515 && defined(AO_HAVE_int_xor_write) 2516 # define AO_int_xor_release_write(addr, val) AO_int_xor_write(addr, val) 2517 # define AO_HAVE_int_xor_release_write 2518 #endif 2519 #if !defined(AO_HAVE_int_xor_release_write) \ 2520 && defined(AO_HAVE_int_xor_release) 2521 # define AO_int_xor_release_write(addr, val) AO_int_xor_release(addr, val) 2522 # define AO_HAVE_int_xor_release_write 2523 #endif 2524 #if !defined(AO_HAVE_int_xor_acquire_read) \ 2525 && defined(AO_HAVE_int_xor_read) 2526 # define AO_int_xor_acquire_read(addr, val) AO_int_xor_read(addr, val) 2527 # define AO_HAVE_int_xor_acquire_read 2528 #endif 2529 #if !defined(AO_HAVE_int_xor_acquire_read) \ 2530 && defined(AO_HAVE_int_xor_acquire) 2531 # define AO_int_xor_acquire_read(addr, val) AO_int_xor_acquire(addr, val) 2532 # define AO_HAVE_int_xor_acquire_read 2533 #endif 2534 2535 /* int_and/or/xor_dd_acquire_read are meaningless. */ 2536 /* 2537 * Copyright (c) 2003-2011 Hewlett-Packard Development Company, L.P. 2538 * 2539 * Permission is hereby granted, free of charge, to any person obtaining a copy 2540 * of this software and associated documentation files (the "Software"), to deal 2541 * in the Software without restriction, including without limitation the rights 2542 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 2543 * copies of the Software, and to permit persons to whom the Software is 2544 * furnished to do so, subject to the following conditions: 2545 * 2546 * The above copyright notice and this permission notice shall be included in 2547 * all copies or substantial portions of the Software. 2548 * 2549 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 2550 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 2551 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 2552 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 2553 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 2554 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 2555 * SOFTWARE. 2556 */ 2557 2558 /* compare_and_swap (based on fetch_compare_and_swap) */ 2559 #if defined(AO_HAVE_fetch_compare_and_swap_full) \ 2560 && !defined(AO_HAVE_compare_and_swap_full) 2561 AO_INLINE int AO_compare_and_swap_full(volatile AO_t * addr,AO_t old_val,AO_t new_val)2562 AO_compare_and_swap_full(volatile AO_t *addr, AO_t old_val, 2563 AO_t new_val) 2564 { 2565 return AO_fetch_compare_and_swap_full(addr, old_val, new_val) 2566 == old_val; 2567 } 2568 # define AO_HAVE_compare_and_swap_full 2569 #endif 2570 2571 #if defined(AO_HAVE_fetch_compare_and_swap_acquire) \ 2572 && !defined(AO_HAVE_compare_and_swap_acquire) 2573 AO_INLINE int AO_compare_and_swap_acquire(volatile AO_t * addr,AO_t old_val,AO_t new_val)2574 AO_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val, 2575 AO_t new_val) 2576 { 2577 return AO_fetch_compare_and_swap_acquire(addr, old_val, new_val) 2578 == old_val; 2579 } 2580 # define AO_HAVE_compare_and_swap_acquire 2581 #endif 2582 2583 #if defined(AO_HAVE_fetch_compare_and_swap_release) \ 2584 && !defined(AO_HAVE_compare_and_swap_release) 2585 AO_INLINE int AO_compare_and_swap_release(volatile AO_t * addr,AO_t old_val,AO_t new_val)2586 AO_compare_and_swap_release(volatile AO_t *addr, AO_t old_val, 2587 AO_t new_val) 2588 { 2589 return AO_fetch_compare_and_swap_release(addr, old_val, new_val) 2590 == old_val; 2591 } 2592 # define AO_HAVE_compare_and_swap_release 2593 #endif 2594 2595 #if defined(AO_HAVE_fetch_compare_and_swap_write) \ 2596 && !defined(AO_HAVE_compare_and_swap_write) 2597 AO_INLINE int AO_compare_and_swap_write(volatile AO_t * addr,AO_t old_val,AO_t new_val)2598 AO_compare_and_swap_write(volatile AO_t *addr, AO_t old_val, 2599 AO_t new_val) 2600 { 2601 return AO_fetch_compare_and_swap_write(addr, old_val, new_val) 2602 == old_val; 2603 } 2604 # define AO_HAVE_compare_and_swap_write 2605 #endif 2606 2607 #if defined(AO_HAVE_fetch_compare_and_swap_read) \ 2608 && !defined(AO_HAVE_compare_and_swap_read) 2609 AO_INLINE int AO_compare_and_swap_read(volatile AO_t * addr,AO_t old_val,AO_t new_val)2610 AO_compare_and_swap_read(volatile AO_t *addr, AO_t old_val, 2611 AO_t new_val) 2612 { 2613 return AO_fetch_compare_and_swap_read(addr, old_val, new_val) 2614 == old_val; 2615 } 2616 # define AO_HAVE_compare_and_swap_read 2617 #endif 2618 2619 #if defined(AO_HAVE_fetch_compare_and_swap) \ 2620 && !defined(AO_HAVE_compare_and_swap) 2621 AO_INLINE int AO_compare_and_swap(volatile AO_t * addr,AO_t old_val,AO_t new_val)2622 AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, 2623 AO_t new_val) 2624 { 2625 return AO_fetch_compare_and_swap(addr, old_val, new_val) == old_val; 2626 } 2627 # define AO_HAVE_compare_and_swap 2628 #endif 2629 2630 #if defined(AO_HAVE_fetch_compare_and_swap_release_write) \ 2631 && !defined(AO_HAVE_compare_and_swap_release_write) 2632 AO_INLINE int AO_compare_and_swap_release_write(volatile AO_t * addr,AO_t old_val,AO_t new_val)2633 AO_compare_and_swap_release_write(volatile AO_t *addr, 2634 AO_t old_val, AO_t new_val) 2635 { 2636 return AO_fetch_compare_and_swap_release_write(addr, old_val, 2637 new_val) == old_val; 2638 } 2639 # define AO_HAVE_compare_and_swap_release_write 2640 #endif 2641 2642 #if defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \ 2643 && !defined(AO_HAVE_compare_and_swap_acquire_read) 2644 AO_INLINE int AO_compare_and_swap_acquire_read(volatile AO_t * addr,AO_t old_val,AO_t new_val)2645 AO_compare_and_swap_acquire_read(volatile AO_t *addr, 2646 AO_t old_val, AO_t new_val) 2647 { 2648 return AO_fetch_compare_and_swap_acquire_read(addr, old_val, 2649 new_val) == old_val; 2650 } 2651 # define AO_HAVE_compare_and_swap_acquire_read 2652 #endif 2653 2654 #if defined(AO_HAVE_fetch_compare_and_swap_dd_acquire_read) \ 2655 && !defined(AO_HAVE_compare_and_swap_dd_acquire_read) 2656 AO_INLINE int AO_compare_and_swap_dd_acquire_read(volatile AO_t * addr,AO_t old_val,AO_t new_val)2657 AO_compare_and_swap_dd_acquire_read(volatile AO_t *addr, 2658 AO_t old_val, AO_t new_val) 2659 { 2660 return AO_fetch_compare_and_swap_dd_acquire_read(addr, old_val, 2661 new_val) == old_val; 2662 } 2663 # define AO_HAVE_compare_and_swap_dd_acquire_read 2664 #endif 2665 2666 /* fetch_and_add */ 2667 /* We first try to implement fetch_and_add variants in terms of the */ 2668 /* corresponding compare_and_swap variants to minimize adding barriers. */ 2669 #if defined(AO_HAVE_compare_and_swap_full) \ 2670 && !defined(AO_HAVE_fetch_and_add_full) 2671 AO_INLINE AO_t AO_fetch_and_add_full(volatile AO_t * addr,AO_t incr)2672 AO_fetch_and_add_full(volatile AO_t *addr, AO_t incr) 2673 { 2674 AO_t old; 2675 2676 do 2677 { 2678 old = *(AO_t *)addr; 2679 } 2680 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, 2681 old + incr))); 2682 return old; 2683 } 2684 # define AO_HAVE_fetch_and_add_full 2685 #endif 2686 2687 #if defined(AO_HAVE_compare_and_swap_acquire) \ 2688 && !defined(AO_HAVE_fetch_and_add_acquire) 2689 AO_INLINE AO_t AO_fetch_and_add_acquire(volatile AO_t * addr,AO_t incr)2690 AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr) 2691 { 2692 AO_t old; 2693 2694 do 2695 { 2696 old = *(AO_t *)addr; 2697 } 2698 while (AO_EXPECT_FALSE(!AO_compare_and_swap_acquire(addr, old, 2699 old + incr))); 2700 return old; 2701 } 2702 # define AO_HAVE_fetch_and_add_acquire 2703 #endif 2704 2705 #if defined(AO_HAVE_compare_and_swap_release) \ 2706 && !defined(AO_HAVE_fetch_and_add_release) 2707 AO_INLINE AO_t AO_fetch_and_add_release(volatile AO_t * addr,AO_t incr)2708 AO_fetch_and_add_release(volatile AO_t *addr, AO_t incr) 2709 { 2710 AO_t old; 2711 2712 do 2713 { 2714 old = *(AO_t *)addr; 2715 } 2716 while (AO_EXPECT_FALSE(!AO_compare_and_swap_release(addr, old, 2717 old + incr))); 2718 return old; 2719 } 2720 # define AO_HAVE_fetch_and_add_release 2721 #endif 2722 2723 #if defined(AO_HAVE_compare_and_swap) \ 2724 && !defined(AO_HAVE_fetch_and_add) 2725 AO_INLINE AO_t AO_fetch_and_add(volatile AO_t * addr,AO_t incr)2726 AO_fetch_and_add(volatile AO_t *addr, AO_t incr) 2727 { 2728 AO_t old; 2729 2730 do 2731 { 2732 old = *(AO_t *)addr; 2733 } 2734 while (AO_EXPECT_FALSE(!AO_compare_and_swap(addr, old, 2735 old + incr))); 2736 return old; 2737 } 2738 # define AO_HAVE_fetch_and_add 2739 #endif 2740 2741 #if defined(AO_HAVE_fetch_and_add_full) 2742 # if !defined(AO_HAVE_fetch_and_add_release) 2743 # define AO_fetch_and_add_release(addr, val) \ 2744 AO_fetch_and_add_full(addr, val) 2745 # define AO_HAVE_fetch_and_add_release 2746 # endif 2747 # if !defined(AO_HAVE_fetch_and_add_acquire) 2748 # define AO_fetch_and_add_acquire(addr, val) \ 2749 AO_fetch_and_add_full(addr, val) 2750 # define AO_HAVE_fetch_and_add_acquire 2751 # endif 2752 # if !defined(AO_HAVE_fetch_and_add_write) 2753 # define AO_fetch_and_add_write(addr, val) \ 2754 AO_fetch_and_add_full(addr, val) 2755 # define AO_HAVE_fetch_and_add_write 2756 # endif 2757 # if !defined(AO_HAVE_fetch_and_add_read) 2758 # define AO_fetch_and_add_read(addr, val) \ 2759 AO_fetch_and_add_full(addr, val) 2760 # define AO_HAVE_fetch_and_add_read 2761 # endif 2762 #endif /* AO_HAVE_fetch_and_add_full */ 2763 2764 #if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \ 2765 && !defined(AO_HAVE_fetch_and_add_acquire) 2766 AO_INLINE AO_t AO_fetch_and_add_acquire(volatile AO_t * addr,AO_t incr)2767 AO_fetch_and_add_acquire(volatile AO_t *addr, AO_t incr) 2768 { 2769 AO_t result = AO_fetch_and_add(addr, incr); 2770 AO_nop_full(); 2771 return result; 2772 } 2773 # define AO_HAVE_fetch_and_add_acquire 2774 #endif 2775 #if defined(AO_HAVE_fetch_and_add) && defined(AO_HAVE_nop_full) \ 2776 && !defined(AO_HAVE_fetch_and_add_release) 2777 # define AO_fetch_and_add_release(addr, incr) \ 2778 (AO_nop_full(), AO_fetch_and_add(addr, incr)) 2779 # define AO_HAVE_fetch_and_add_release 2780 #endif 2781 2782 #if !defined(AO_HAVE_fetch_and_add) \ 2783 && defined(AO_HAVE_fetch_and_add_release) 2784 # define AO_fetch_and_add(addr, val) \ 2785 AO_fetch_and_add_release(addr, val) 2786 # define AO_HAVE_fetch_and_add 2787 #endif 2788 #if !defined(AO_HAVE_fetch_and_add) \ 2789 && defined(AO_HAVE_fetch_and_add_acquire) 2790 # define AO_fetch_and_add(addr, val) \ 2791 AO_fetch_and_add_acquire(addr, val) 2792 # define AO_HAVE_fetch_and_add 2793 #endif 2794 #if !defined(AO_HAVE_fetch_and_add) \ 2795 && defined(AO_HAVE_fetch_and_add_write) 2796 # define AO_fetch_and_add(addr, val) \ 2797 AO_fetch_and_add_write(addr, val) 2798 # define AO_HAVE_fetch_and_add 2799 #endif 2800 #if !defined(AO_HAVE_fetch_and_add) \ 2801 && defined(AO_HAVE_fetch_and_add_read) 2802 # define AO_fetch_and_add(addr, val) \ 2803 AO_fetch_and_add_read(addr, val) 2804 # define AO_HAVE_fetch_and_add 2805 #endif 2806 2807 #if defined(AO_HAVE_fetch_and_add_acquire) \ 2808 && defined(AO_HAVE_nop_full) && !defined(AO_HAVE_fetch_and_add_full) 2809 # define AO_fetch_and_add_full(addr, val) \ 2810 (AO_nop_full(), AO_fetch_and_add_acquire(addr, val)) 2811 # define AO_HAVE_fetch_and_add_full 2812 #endif 2813 2814 #if !defined(AO_HAVE_fetch_and_add_release_write) \ 2815 && defined(AO_HAVE_fetch_and_add_write) 2816 # define AO_fetch_and_add_release_write(addr, val) \ 2817 AO_fetch_and_add_write(addr, val) 2818 # define AO_HAVE_fetch_and_add_release_write 2819 #endif 2820 #if !defined(AO_HAVE_fetch_and_add_release_write) \ 2821 && defined(AO_HAVE_fetch_and_add_release) 2822 # define AO_fetch_and_add_release_write(addr, val) \ 2823 AO_fetch_and_add_release(addr, val) 2824 # define AO_HAVE_fetch_and_add_release_write 2825 #endif 2826 2827 #if !defined(AO_HAVE_fetch_and_add_acquire_read) \ 2828 && defined(AO_HAVE_fetch_and_add_read) 2829 # define AO_fetch_and_add_acquire_read(addr, val) \ 2830 AO_fetch_and_add_read(addr, val) 2831 # define AO_HAVE_fetch_and_add_acquire_read 2832 #endif 2833 #if !defined(AO_HAVE_fetch_and_add_acquire_read) \ 2834 && defined(AO_HAVE_fetch_and_add_acquire) 2835 # define AO_fetch_and_add_acquire_read(addr, val) \ 2836 AO_fetch_and_add_acquire(addr, val) 2837 # define AO_HAVE_fetch_and_add_acquire_read 2838 #endif 2839 2840 #ifdef AO_NO_DD_ORDERING 2841 # if defined(AO_HAVE_fetch_and_add_acquire_read) 2842 # define AO_fetch_and_add_dd_acquire_read(addr, val) \ 2843 AO_fetch_and_add_acquire_read(addr, val) 2844 # define AO_HAVE_fetch_and_add_dd_acquire_read 2845 # endif 2846 #else 2847 # if defined(AO_HAVE_fetch_and_add) 2848 # define AO_fetch_and_add_dd_acquire_read(addr, val) \ 2849 AO_fetch_and_add(addr, val) 2850 # define AO_HAVE_fetch_and_add_dd_acquire_read 2851 # endif 2852 #endif /* !AO_NO_DD_ORDERING */ 2853 2854 /* fetch_and_add1 */ 2855 #if defined(AO_HAVE_fetch_and_add_full) \ 2856 && !defined(AO_HAVE_fetch_and_add1_full) 2857 # define AO_fetch_and_add1_full(addr) \ 2858 AO_fetch_and_add_full(addr, 1) 2859 # define AO_HAVE_fetch_and_add1_full 2860 #endif 2861 #if defined(AO_HAVE_fetch_and_add_release) \ 2862 && !defined(AO_HAVE_fetch_and_add1_release) 2863 # define AO_fetch_and_add1_release(addr) \ 2864 AO_fetch_and_add_release(addr, 1) 2865 # define AO_HAVE_fetch_and_add1_release 2866 #endif 2867 #if defined(AO_HAVE_fetch_and_add_acquire) \ 2868 && !defined(AO_HAVE_fetch_and_add1_acquire) 2869 # define AO_fetch_and_add1_acquire(addr) \ 2870 AO_fetch_and_add_acquire(addr, 1) 2871 # define AO_HAVE_fetch_and_add1_acquire 2872 #endif 2873 #if defined(AO_HAVE_fetch_and_add_write) \ 2874 && !defined(AO_HAVE_fetch_and_add1_write) 2875 # define AO_fetch_and_add1_write(addr) \ 2876 AO_fetch_and_add_write(addr, 1) 2877 # define AO_HAVE_fetch_and_add1_write 2878 #endif 2879 #if defined(AO_HAVE_fetch_and_add_read) \ 2880 && !defined(AO_HAVE_fetch_and_add1_read) 2881 # define AO_fetch_and_add1_read(addr) \ 2882 AO_fetch_and_add_read(addr, 1) 2883 # define AO_HAVE_fetch_and_add1_read 2884 #endif 2885 #if defined(AO_HAVE_fetch_and_add_release_write) \ 2886 && !defined(AO_HAVE_fetch_and_add1_release_write) 2887 # define AO_fetch_and_add1_release_write(addr) \ 2888 AO_fetch_and_add_release_write(addr, 1) 2889 # define AO_HAVE_fetch_and_add1_release_write 2890 #endif 2891 #if defined(AO_HAVE_fetch_and_add_acquire_read) \ 2892 && !defined(AO_HAVE_fetch_and_add1_acquire_read) 2893 # define AO_fetch_and_add1_acquire_read(addr) \ 2894 AO_fetch_and_add_acquire_read(addr, 1) 2895 # define AO_HAVE_fetch_and_add1_acquire_read 2896 #endif 2897 #if defined(AO_HAVE_fetch_and_add) \ 2898 && !defined(AO_HAVE_fetch_and_add1) 2899 # define AO_fetch_and_add1(addr) AO_fetch_and_add(addr, 1) 2900 # define AO_HAVE_fetch_and_add1 2901 #endif 2902 2903 #if defined(AO_HAVE_fetch_and_add1_full) 2904 # if !defined(AO_HAVE_fetch_and_add1_release) 2905 # define AO_fetch_and_add1_release(addr) \ 2906 AO_fetch_and_add1_full(addr) 2907 # define AO_HAVE_fetch_and_add1_release 2908 # endif 2909 # if !defined(AO_HAVE_fetch_and_add1_acquire) 2910 # define AO_fetch_and_add1_acquire(addr) \ 2911 AO_fetch_and_add1_full(addr) 2912 # define AO_HAVE_fetch_and_add1_acquire 2913 # endif 2914 # if !defined(AO_HAVE_fetch_and_add1_write) 2915 # define AO_fetch_and_add1_write(addr) \ 2916 AO_fetch_and_add1_full(addr) 2917 # define AO_HAVE_fetch_and_add1_write 2918 # endif 2919 # if !defined(AO_HAVE_fetch_and_add1_read) 2920 # define AO_fetch_and_add1_read(addr) \ 2921 AO_fetch_and_add1_full(addr) 2922 # define AO_HAVE_fetch_and_add1_read 2923 # endif 2924 #endif /* AO_HAVE_fetch_and_add1_full */ 2925 2926 #if !defined(AO_HAVE_fetch_and_add1) \ 2927 && defined(AO_HAVE_fetch_and_add1_release) 2928 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_release(addr) 2929 # define AO_HAVE_fetch_and_add1 2930 #endif 2931 #if !defined(AO_HAVE_fetch_and_add1) \ 2932 && defined(AO_HAVE_fetch_and_add1_acquire) 2933 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_acquire(addr) 2934 # define AO_HAVE_fetch_and_add1 2935 #endif 2936 #if !defined(AO_HAVE_fetch_and_add1) \ 2937 && defined(AO_HAVE_fetch_and_add1_write) 2938 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_write(addr) 2939 # define AO_HAVE_fetch_and_add1 2940 #endif 2941 #if !defined(AO_HAVE_fetch_and_add1) \ 2942 && defined(AO_HAVE_fetch_and_add1_read) 2943 # define AO_fetch_and_add1(addr) AO_fetch_and_add1_read(addr) 2944 # define AO_HAVE_fetch_and_add1 2945 #endif 2946 2947 #if defined(AO_HAVE_fetch_and_add1_acquire) \ 2948 && defined(AO_HAVE_nop_full) \ 2949 && !defined(AO_HAVE_fetch_and_add1_full) 2950 # define AO_fetch_and_add1_full(addr) \ 2951 (AO_nop_full(), AO_fetch_and_add1_acquire(addr)) 2952 # define AO_HAVE_fetch_and_add1_full 2953 #endif 2954 2955 #if !defined(AO_HAVE_fetch_and_add1_release_write) \ 2956 && defined(AO_HAVE_fetch_and_add1_write) 2957 # define AO_fetch_and_add1_release_write(addr) \ 2958 AO_fetch_and_add1_write(addr) 2959 # define AO_HAVE_fetch_and_add1_release_write 2960 #endif 2961 #if !defined(AO_HAVE_fetch_and_add1_release_write) \ 2962 && defined(AO_HAVE_fetch_and_add1_release) 2963 # define AO_fetch_and_add1_release_write(addr) \ 2964 AO_fetch_and_add1_release(addr) 2965 # define AO_HAVE_fetch_and_add1_release_write 2966 #endif 2967 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) \ 2968 && defined(AO_HAVE_fetch_and_add1_read) 2969 # define AO_fetch_and_add1_acquire_read(addr) \ 2970 AO_fetch_and_add1_read(addr) 2971 # define AO_HAVE_fetch_and_add1_acquire_read 2972 #endif 2973 #if !defined(AO_HAVE_fetch_and_add1_acquire_read) \ 2974 && defined(AO_HAVE_fetch_and_add1_acquire) 2975 # define AO_fetch_and_add1_acquire_read(addr) \ 2976 AO_fetch_and_add1_acquire(addr) 2977 # define AO_HAVE_fetch_and_add1_acquire_read 2978 #endif 2979 2980 #ifdef AO_NO_DD_ORDERING 2981 # if defined(AO_HAVE_fetch_and_add1_acquire_read) 2982 # define AO_fetch_and_add1_dd_acquire_read(addr) \ 2983 AO_fetch_and_add1_acquire_read(addr) 2984 # define AO_HAVE_fetch_and_add1_dd_acquire_read 2985 # endif 2986 #else 2987 # if defined(AO_HAVE_fetch_and_add1) 2988 # define AO_fetch_and_add1_dd_acquire_read(addr) \ 2989 AO_fetch_and_add1(addr) 2990 # define AO_HAVE_fetch_and_add1_dd_acquire_read 2991 # endif 2992 #endif /* !AO_NO_DD_ORDERING */ 2993 2994 /* fetch_and_sub1 */ 2995 #if defined(AO_HAVE_fetch_and_add_full) \ 2996 && !defined(AO_HAVE_fetch_and_sub1_full) 2997 # define AO_fetch_and_sub1_full(addr) \ 2998 AO_fetch_and_add_full(addr, (AO_t)(-1)) 2999 # define AO_HAVE_fetch_and_sub1_full 3000 #endif 3001 #if defined(AO_HAVE_fetch_and_add_release) \ 3002 && !defined(AO_HAVE_fetch_and_sub1_release) 3003 # define AO_fetch_and_sub1_release(addr) \ 3004 AO_fetch_and_add_release(addr, (AO_t)(-1)) 3005 # define AO_HAVE_fetch_and_sub1_release 3006 #endif 3007 #if defined(AO_HAVE_fetch_and_add_acquire) \ 3008 && !defined(AO_HAVE_fetch_and_sub1_acquire) 3009 # define AO_fetch_and_sub1_acquire(addr) \ 3010 AO_fetch_and_add_acquire(addr, (AO_t)(-1)) 3011 # define AO_HAVE_fetch_and_sub1_acquire 3012 #endif 3013 #if defined(AO_HAVE_fetch_and_add_write) \ 3014 && !defined(AO_HAVE_fetch_and_sub1_write) 3015 # define AO_fetch_and_sub1_write(addr) \ 3016 AO_fetch_and_add_write(addr, (AO_t)(-1)) 3017 # define AO_HAVE_fetch_and_sub1_write 3018 #endif 3019 #if defined(AO_HAVE_fetch_and_add_read) \ 3020 && !defined(AO_HAVE_fetch_and_sub1_read) 3021 # define AO_fetch_and_sub1_read(addr) \ 3022 AO_fetch_and_add_read(addr, (AO_t)(-1)) 3023 # define AO_HAVE_fetch_and_sub1_read 3024 #endif 3025 #if defined(AO_HAVE_fetch_and_add_release_write) \ 3026 && !defined(AO_HAVE_fetch_and_sub1_release_write) 3027 # define AO_fetch_and_sub1_release_write(addr) \ 3028 AO_fetch_and_add_release_write(addr, (AO_t)(-1)) 3029 # define AO_HAVE_fetch_and_sub1_release_write 3030 #endif 3031 #if defined(AO_HAVE_fetch_and_add_acquire_read) \ 3032 && !defined(AO_HAVE_fetch_and_sub1_acquire_read) 3033 # define AO_fetch_and_sub1_acquire_read(addr) \ 3034 AO_fetch_and_add_acquire_read(addr, (AO_t)(-1)) 3035 # define AO_HAVE_fetch_and_sub1_acquire_read 3036 #endif 3037 #if defined(AO_HAVE_fetch_and_add) \ 3038 && !defined(AO_HAVE_fetch_and_sub1) 3039 # define AO_fetch_and_sub1(addr) \ 3040 AO_fetch_and_add(addr, (AO_t)(-1)) 3041 # define AO_HAVE_fetch_and_sub1 3042 #endif 3043 3044 #if defined(AO_HAVE_fetch_and_sub1_full) 3045 # if !defined(AO_HAVE_fetch_and_sub1_release) 3046 # define AO_fetch_and_sub1_release(addr) \ 3047 AO_fetch_and_sub1_full(addr) 3048 # define AO_HAVE_fetch_and_sub1_release 3049 # endif 3050 # if !defined(AO_HAVE_fetch_and_sub1_acquire) 3051 # define AO_fetch_and_sub1_acquire(addr) \ 3052 AO_fetch_and_sub1_full(addr) 3053 # define AO_HAVE_fetch_and_sub1_acquire 3054 # endif 3055 # if !defined(AO_HAVE_fetch_and_sub1_write) 3056 # define AO_fetch_and_sub1_write(addr) \ 3057 AO_fetch_and_sub1_full(addr) 3058 # define AO_HAVE_fetch_and_sub1_write 3059 # endif 3060 # if !defined(AO_HAVE_fetch_and_sub1_read) 3061 # define AO_fetch_and_sub1_read(addr) \ 3062 AO_fetch_and_sub1_full(addr) 3063 # define AO_HAVE_fetch_and_sub1_read 3064 # endif 3065 #endif /* AO_HAVE_fetch_and_sub1_full */ 3066 3067 #if !defined(AO_HAVE_fetch_and_sub1) \ 3068 && defined(AO_HAVE_fetch_and_sub1_release) 3069 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_release(addr) 3070 # define AO_HAVE_fetch_and_sub1 3071 #endif 3072 #if !defined(AO_HAVE_fetch_and_sub1) \ 3073 && defined(AO_HAVE_fetch_and_sub1_acquire) 3074 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_acquire(addr) 3075 # define AO_HAVE_fetch_and_sub1 3076 #endif 3077 #if !defined(AO_HAVE_fetch_and_sub1) \ 3078 && defined(AO_HAVE_fetch_and_sub1_write) 3079 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_write(addr) 3080 # define AO_HAVE_fetch_and_sub1 3081 #endif 3082 #if !defined(AO_HAVE_fetch_and_sub1) \ 3083 && defined(AO_HAVE_fetch_and_sub1_read) 3084 # define AO_fetch_and_sub1(addr) AO_fetch_and_sub1_read(addr) 3085 # define AO_HAVE_fetch_and_sub1 3086 #endif 3087 3088 #if defined(AO_HAVE_fetch_and_sub1_acquire) \ 3089 && defined(AO_HAVE_nop_full) \ 3090 && !defined(AO_HAVE_fetch_and_sub1_full) 3091 # define AO_fetch_and_sub1_full(addr) \ 3092 (AO_nop_full(), AO_fetch_and_sub1_acquire(addr)) 3093 # define AO_HAVE_fetch_and_sub1_full 3094 #endif 3095 3096 #if !defined(AO_HAVE_fetch_and_sub1_release_write) \ 3097 && defined(AO_HAVE_fetch_and_sub1_write) 3098 # define AO_fetch_and_sub1_release_write(addr) \ 3099 AO_fetch_and_sub1_write(addr) 3100 # define AO_HAVE_fetch_and_sub1_release_write 3101 #endif 3102 #if !defined(AO_HAVE_fetch_and_sub1_release_write) \ 3103 && defined(AO_HAVE_fetch_and_sub1_release) 3104 # define AO_fetch_and_sub1_release_write(addr) \ 3105 AO_fetch_and_sub1_release(addr) 3106 # define AO_HAVE_fetch_and_sub1_release_write 3107 #endif 3108 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \ 3109 && defined(AO_HAVE_fetch_and_sub1_read) 3110 # define AO_fetch_and_sub1_acquire_read(addr) \ 3111 AO_fetch_and_sub1_read(addr) 3112 # define AO_HAVE_fetch_and_sub1_acquire_read 3113 #endif 3114 #if !defined(AO_HAVE_fetch_and_sub1_acquire_read) \ 3115 && defined(AO_HAVE_fetch_and_sub1_acquire) 3116 # define AO_fetch_and_sub1_acquire_read(addr) \ 3117 AO_fetch_and_sub1_acquire(addr) 3118 # define AO_HAVE_fetch_and_sub1_acquire_read 3119 #endif 3120 3121 #ifdef AO_NO_DD_ORDERING 3122 # if defined(AO_HAVE_fetch_and_sub1_acquire_read) 3123 # define AO_fetch_and_sub1_dd_acquire_read(addr) \ 3124 AO_fetch_and_sub1_acquire_read(addr) 3125 # define AO_HAVE_fetch_and_sub1_dd_acquire_read 3126 # endif 3127 #else 3128 # if defined(AO_HAVE_fetch_and_sub1) 3129 # define AO_fetch_and_sub1_dd_acquire_read(addr) \ 3130 AO_fetch_and_sub1(addr) 3131 # define AO_HAVE_fetch_and_sub1_dd_acquire_read 3132 # endif 3133 #endif /* !AO_NO_DD_ORDERING */ 3134 3135 /* and */ 3136 #if defined(AO_HAVE_compare_and_swap_full) \ 3137 && !defined(AO_HAVE_and_full) 3138 AO_INLINE void AO_and_full(volatile AO_t * addr,AO_t value)3139 AO_and_full(volatile AO_t *addr, AO_t value) 3140 { 3141 AO_t old; 3142 3143 do 3144 { 3145 old = *(AO_t *)addr; 3146 } 3147 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, 3148 old & value))); 3149 } 3150 # define AO_HAVE_and_full 3151 #endif 3152 3153 #if defined(AO_HAVE_and_full) 3154 # if !defined(AO_HAVE_and_release) 3155 # define AO_and_release(addr, val) AO_and_full(addr, val) 3156 # define AO_HAVE_and_release 3157 # endif 3158 # if !defined(AO_HAVE_and_acquire) 3159 # define AO_and_acquire(addr, val) AO_and_full(addr, val) 3160 # define AO_HAVE_and_acquire 3161 # endif 3162 # if !defined(AO_HAVE_and_write) 3163 # define AO_and_write(addr, val) AO_and_full(addr, val) 3164 # define AO_HAVE_and_write 3165 # endif 3166 # if !defined(AO_HAVE_and_read) 3167 # define AO_and_read(addr, val) AO_and_full(addr, val) 3168 # define AO_HAVE_and_read 3169 # endif 3170 #endif /* AO_HAVE_and_full */ 3171 3172 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_release) 3173 # define AO_and(addr, val) AO_and_release(addr, val) 3174 # define AO_HAVE_and 3175 #endif 3176 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_acquire) 3177 # define AO_and(addr, val) AO_and_acquire(addr, val) 3178 # define AO_HAVE_and 3179 #endif 3180 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_write) 3181 # define AO_and(addr, val) AO_and_write(addr, val) 3182 # define AO_HAVE_and 3183 #endif 3184 #if !defined(AO_HAVE_and) && defined(AO_HAVE_and_read) 3185 # define AO_and(addr, val) AO_and_read(addr, val) 3186 # define AO_HAVE_and 3187 #endif 3188 3189 #if defined(AO_HAVE_and_acquire) && defined(AO_HAVE_nop_full) \ 3190 && !defined(AO_HAVE_and_full) 3191 # define AO_and_full(addr, val) \ 3192 (AO_nop_full(), AO_and_acquire(addr, val)) 3193 # define AO_HAVE_and_full 3194 #endif 3195 3196 #if !defined(AO_HAVE_and_release_write) \ 3197 && defined(AO_HAVE_and_write) 3198 # define AO_and_release_write(addr, val) AO_and_write(addr, val) 3199 # define AO_HAVE_and_release_write 3200 #endif 3201 #if !defined(AO_HAVE_and_release_write) \ 3202 && defined(AO_HAVE_and_release) 3203 # define AO_and_release_write(addr, val) AO_and_release(addr, val) 3204 # define AO_HAVE_and_release_write 3205 #endif 3206 #if !defined(AO_HAVE_and_acquire_read) \ 3207 && defined(AO_HAVE_and_read) 3208 # define AO_and_acquire_read(addr, val) AO_and_read(addr, val) 3209 # define AO_HAVE_and_acquire_read 3210 #endif 3211 #if !defined(AO_HAVE_and_acquire_read) \ 3212 && defined(AO_HAVE_and_acquire) 3213 # define AO_and_acquire_read(addr, val) AO_and_acquire(addr, val) 3214 # define AO_HAVE_and_acquire_read 3215 #endif 3216 3217 /* or */ 3218 #if defined(AO_HAVE_compare_and_swap_full) \ 3219 && !defined(AO_HAVE_or_full) 3220 AO_INLINE void AO_or_full(volatile AO_t * addr,AO_t value)3221 AO_or_full(volatile AO_t *addr, AO_t value) 3222 { 3223 AO_t old; 3224 3225 do 3226 { 3227 old = *(AO_t *)addr; 3228 } 3229 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, 3230 old | value))); 3231 } 3232 # define AO_HAVE_or_full 3233 #endif 3234 3235 #if defined(AO_HAVE_or_full) 3236 # if !defined(AO_HAVE_or_release) 3237 # define AO_or_release(addr, val) AO_or_full(addr, val) 3238 # define AO_HAVE_or_release 3239 # endif 3240 # if !defined(AO_HAVE_or_acquire) 3241 # define AO_or_acquire(addr, val) AO_or_full(addr, val) 3242 # define AO_HAVE_or_acquire 3243 # endif 3244 # if !defined(AO_HAVE_or_write) 3245 # define AO_or_write(addr, val) AO_or_full(addr, val) 3246 # define AO_HAVE_or_write 3247 # endif 3248 # if !defined(AO_HAVE_or_read) 3249 # define AO_or_read(addr, val) AO_or_full(addr, val) 3250 # define AO_HAVE_or_read 3251 # endif 3252 #endif /* AO_HAVE_or_full */ 3253 3254 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_release) 3255 # define AO_or(addr, val) AO_or_release(addr, val) 3256 # define AO_HAVE_or 3257 #endif 3258 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_acquire) 3259 # define AO_or(addr, val) AO_or_acquire(addr, val) 3260 # define AO_HAVE_or 3261 #endif 3262 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_write) 3263 # define AO_or(addr, val) AO_or_write(addr, val) 3264 # define AO_HAVE_or 3265 #endif 3266 #if !defined(AO_HAVE_or) && defined(AO_HAVE_or_read) 3267 # define AO_or(addr, val) AO_or_read(addr, val) 3268 # define AO_HAVE_or 3269 #endif 3270 3271 #if defined(AO_HAVE_or_acquire) && defined(AO_HAVE_nop_full) \ 3272 && !defined(AO_HAVE_or_full) 3273 # define AO_or_full(addr, val) \ 3274 (AO_nop_full(), AO_or_acquire(addr, val)) 3275 # define AO_HAVE_or_full 3276 #endif 3277 3278 #if !defined(AO_HAVE_or_release_write) \ 3279 && defined(AO_HAVE_or_write) 3280 # define AO_or_release_write(addr, val) AO_or_write(addr, val) 3281 # define AO_HAVE_or_release_write 3282 #endif 3283 #if !defined(AO_HAVE_or_release_write) \ 3284 && defined(AO_HAVE_or_release) 3285 # define AO_or_release_write(addr, val) AO_or_release(addr, val) 3286 # define AO_HAVE_or_release_write 3287 #endif 3288 #if !defined(AO_HAVE_or_acquire_read) && defined(AO_HAVE_or_read) 3289 # define AO_or_acquire_read(addr, val) AO_or_read(addr, val) 3290 # define AO_HAVE_or_acquire_read 3291 #endif 3292 #if !defined(AO_HAVE_or_acquire_read) \ 3293 && defined(AO_HAVE_or_acquire) 3294 # define AO_or_acquire_read(addr, val) AO_or_acquire(addr, val) 3295 # define AO_HAVE_or_acquire_read 3296 #endif 3297 3298 /* xor */ 3299 #if defined(AO_HAVE_compare_and_swap_full) \ 3300 && !defined(AO_HAVE_xor_full) 3301 AO_INLINE void AO_xor_full(volatile AO_t * addr,AO_t value)3302 AO_xor_full(volatile AO_t *addr, AO_t value) 3303 { 3304 AO_t old; 3305 3306 do 3307 { 3308 old = *(AO_t *)addr; 3309 } 3310 while (AO_EXPECT_FALSE(!AO_compare_and_swap_full(addr, old, 3311 old ^ value))); 3312 } 3313 # define AO_HAVE_xor_full 3314 #endif 3315 3316 #if defined(AO_HAVE_xor_full) 3317 # if !defined(AO_HAVE_xor_release) 3318 # define AO_xor_release(addr, val) AO_xor_full(addr, val) 3319 # define AO_HAVE_xor_release 3320 # endif 3321 # if !defined(AO_HAVE_xor_acquire) 3322 # define AO_xor_acquire(addr, val) AO_xor_full(addr, val) 3323 # define AO_HAVE_xor_acquire 3324 # endif 3325 # if !defined(AO_HAVE_xor_write) 3326 # define AO_xor_write(addr, val) AO_xor_full(addr, val) 3327 # define AO_HAVE_xor_write 3328 # endif 3329 # if !defined(AO_HAVE_xor_read) 3330 # define AO_xor_read(addr, val) AO_xor_full(addr, val) 3331 # define AO_HAVE_xor_read 3332 # endif 3333 #endif /* AO_HAVE_xor_full */ 3334 3335 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_release) 3336 # define AO_xor(addr, val) AO_xor_release(addr, val) 3337 # define AO_HAVE_xor 3338 #endif 3339 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_acquire) 3340 # define AO_xor(addr, val) AO_xor_acquire(addr, val) 3341 # define AO_HAVE_xor 3342 #endif 3343 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_write) 3344 # define AO_xor(addr, val) AO_xor_write(addr, val) 3345 # define AO_HAVE_xor 3346 #endif 3347 #if !defined(AO_HAVE_xor) && defined(AO_HAVE_xor_read) 3348 # define AO_xor(addr, val) AO_xor_read(addr, val) 3349 # define AO_HAVE_xor 3350 #endif 3351 3352 #if defined(AO_HAVE_xor_acquire) && defined(AO_HAVE_nop_full) \ 3353 && !defined(AO_HAVE_xor_full) 3354 # define AO_xor_full(addr, val) \ 3355 (AO_nop_full(), AO_xor_acquire(addr, val)) 3356 # define AO_HAVE_xor_full 3357 #endif 3358 3359 #if !defined(AO_HAVE_xor_release_write) \ 3360 && defined(AO_HAVE_xor_write) 3361 # define AO_xor_release_write(addr, val) AO_xor_write(addr, val) 3362 # define AO_HAVE_xor_release_write 3363 #endif 3364 #if !defined(AO_HAVE_xor_release_write) \ 3365 && defined(AO_HAVE_xor_release) 3366 # define AO_xor_release_write(addr, val) AO_xor_release(addr, val) 3367 # define AO_HAVE_xor_release_write 3368 #endif 3369 #if !defined(AO_HAVE_xor_acquire_read) \ 3370 && defined(AO_HAVE_xor_read) 3371 # define AO_xor_acquire_read(addr, val) AO_xor_read(addr, val) 3372 # define AO_HAVE_xor_acquire_read 3373 #endif 3374 #if !defined(AO_HAVE_xor_acquire_read) \ 3375 && defined(AO_HAVE_xor_acquire) 3376 # define AO_xor_acquire_read(addr, val) AO_xor_acquire(addr, val) 3377 # define AO_HAVE_xor_acquire_read 3378 #endif 3379 3380 /* and/or/xor_dd_acquire_read are meaningless. */ 3381