1 /* 2 * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers 3 * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. 4 * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. 5 * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved. 6 * 7 * 8 * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED 9 * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. 10 * 11 * Permission is hereby granted to use or copy this program 12 * for any purpose, provided the above notices are retained on all copies. 13 * Permission to modify the code and to distribute modified code is granted, 14 * provided the above notices are retained, and a notice that the code was 15 * modified is included with the above copyright notice. 16 */ 17 18 #ifndef GC_LOCKS_H 19 #define GC_LOCKS_H 20 21 /* 22 * Mutual exclusion between allocator/collector routines. 23 * Needed if there is more than one allocator thread. 24 * FASTLOCK() is assumed to try to acquire the lock in a cheap and 25 * dirty way that is acceptable for a few instructions, e.g. by 26 * inhibiting preemption. This is assumed to have succeeded only 27 * if a subsequent call to FASTLOCK_SUCCEEDED() returns TRUE. 28 * FASTUNLOCK() is called whether or not FASTLOCK_SUCCEEDED(). 29 * If signals cannot be tolerated with the FASTLOCK held, then 30 * FASTLOCK should disable signals. The code executed under 31 * FASTLOCK is otherwise immune to interruption, provided it is 32 * not restarted. 33 * DCL_LOCK_STATE declares any local variables needed by LOCK and UNLOCK 34 * and/or DISABLE_SIGNALS and ENABLE_SIGNALS and/or FASTLOCK. 35 * (There is currently no equivalent for FASTLOCK.) 36 * 37 * In the PARALLEL_MARK case, we also need to define a number of 38 * other inline finctions here: 39 * GC_bool GC_compare_and_exchange( volatile GC_word *addr, 40 * GC_word old, GC_word new ) 41 * GC_word GC_atomic_add( volatile GC_word *addr, GC_word how_much ) 42 * void GC_memory_barrier( ) 43 * 44 */ 45 # ifdef THREADS 46 void GC_noop1 GC_PROTO((word)); 47 # ifdef PCR_OBSOLETE /* Faster, but broken with multiple lwp's */ 48 # include "th/PCR_Th.h" 49 # include "th/PCR_ThCrSec.h" 50 extern struct PCR_Th_MLRep GC_allocate_ml; 51 # define DCL_LOCK_STATE PCR_sigset_t GC_old_sig_mask 52 # define LOCK() PCR_Th_ML_Acquire(&GC_allocate_ml) 53 # define UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml) 54 # define UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml) 55 # define FASTLOCK() PCR_ThCrSec_EnterSys() 56 /* Here we cheat (a lot): */ 57 # define FASTLOCK_SUCCEEDED() (*(int *)(&GC_allocate_ml) == 0) 58 /* TRUE if nobody currently holds the lock */ 59 # define FASTUNLOCK() PCR_ThCrSec_ExitSys() 60 # endif 61 # ifdef PCR 62 # include <base/PCR_Base.h> 63 # include <th/PCR_Th.h> 64 extern PCR_Th_ML GC_allocate_ml; 65 # define DCL_LOCK_STATE \ 66 PCR_ERes GC_fastLockRes; PCR_sigset_t GC_old_sig_mask 67 # define LOCK() PCR_Th_ML_Acquire(&GC_allocate_ml) 68 # define UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml) 69 # define FASTLOCK() (GC_fastLockRes = PCR_Th_ML_Try(&GC_allocate_ml)) 70 # define FASTLOCK_SUCCEEDED() (GC_fastLockRes == PCR_ERes_okay) 71 # define FASTUNLOCK() {\ 72 if( FASTLOCK_SUCCEEDED() ) PCR_Th_ML_Release(&GC_allocate_ml); } 73 # endif 74 # ifdef SRC_M3 75 extern GC_word RT0u__inCritical; 76 # define LOCK() RT0u__inCritical++ 77 # define UNLOCK() RT0u__inCritical-- 78 # endif 79 # ifdef SN_TARGET_PS3 80 # include <pthread.h> 81 extern pthread_mutex_t GC_allocate_ml; 82 # define LOCK() pthread_mutex_lock(&GC_allocate_ml) 83 # define UNLOCK() pthread_mutex_unlock(&GC_allocate_ml) 84 # endif 85 # ifdef GC_SOLARIS_THREADS 86 # include <thread.h> 87 # include <signal.h> 88 extern mutex_t GC_allocate_ml; 89 # define LOCK() mutex_lock(&GC_allocate_ml); 90 # define UNLOCK() mutex_unlock(&GC_allocate_ml); 91 # endif 92 93 /* Try to define GC_TEST_AND_SET and a matching GC_CLEAR for spin lock */ 94 /* acquisition and release. We need this for correct operation of the */ 95 /* incremental GC. */ 96 # ifdef __GNUC__ 97 # if defined(I386) GC_test_and_set(volatile unsigned int * addr)98 inline static int GC_test_and_set(volatile unsigned int *addr) { 99 int oldval; 100 /* Note: the "xchg" instruction does not need a "lock" prefix */ 101 __asm__ __volatile__("xchgl %0, %1" 102 : "=r"(oldval), "=m"(*(addr)) 103 : "0"(1), "m"(*(addr)) : "memory"); 104 return oldval; 105 } 106 # define GC_TEST_AND_SET_DEFINED 107 # endif 108 # if defined(IA64) 109 # if defined(__INTEL_COMPILER) 110 # include <ia64intrin.h> 111 # endif GC_test_and_set(volatile unsigned int * addr)112 inline static int GC_test_and_set(volatile unsigned int *addr) { 113 long oldval, n = 1; 114 # ifndef __INTEL_COMPILER 115 __asm__ __volatile__("xchg4 %0=%1,%2" 116 : "=r"(oldval), "=m"(*addr) 117 : "r"(n) : "memory"); 118 # else 119 oldval = _InterlockedExchange(addr, n); 120 # endif 121 return oldval; 122 } 123 # define GC_TEST_AND_SET_DEFINED 124 /* Should this handle post-increment addressing?? */ GC_clear(volatile unsigned int * addr)125 inline static void GC_clear(volatile unsigned int *addr) { 126 # ifndef __INTEL_COMPILER 127 __asm__ __volatile__("st4.rel %0=r0" : "=m" (*addr) : : "memory"); 128 # else 129 // there is no st4 but I can use xchg I hope 130 _InterlockedExchange(addr, 0); 131 # endif 132 } 133 # define GC_CLEAR_DEFINED 134 # endif 135 # ifdef SPARC GC_test_and_set(volatile unsigned int * addr)136 inline static int GC_test_and_set(volatile unsigned int *addr) { 137 int oldval; 138 139 __asm__ __volatile__("ldstub %1,%0" 140 : "=r"(oldval), "=m"(*addr) 141 : "m"(*addr) : "memory"); 142 return oldval; 143 } 144 # define GC_TEST_AND_SET_DEFINED 145 # endif 146 # ifdef M68K 147 /* Contributed by Tony Mantler. I'm not sure how well it was */ 148 /* tested. */ GC_test_and_set(volatile unsigned int * addr)149 inline static int GC_test_and_set(volatile unsigned int *addr) { 150 char oldval; /* this must be no longer than 8 bits */ 151 152 /* The return value is semi-phony. */ 153 /* 'tas' sets bit 7 while the return */ 154 /* value pretends bit 0 was set */ 155 __asm__ __volatile__( 156 "tas %1@; sne %0; negb %0" 157 : "=d" (oldval) 158 : "a" (addr) : "memory"); 159 return oldval; 160 } 161 # define GC_TEST_AND_SET_DEFINED 162 # endif 163 # if defined(POWERPC) GC_test_and_set(volatile unsigned int * addr)164 inline static int GC_test_and_set(volatile unsigned int *addr) { 165 int oldval; 166 int temp = 1; /* locked value */ 167 168 __asm__ __volatile__( 169 "1:\tlwarx %0,0,%1\n" /* load and reserve */ 170 "\tcmpwi %0, 0\n" /* if load is */ 171 "\tbne 2f\n" /* non-zero, return already set */ 172 "\tstwcx. %2,0,%1\n" /* else store conditional */ 173 "\tbne- 1b\n" /* retry if lost reservation */ 174 "\tsync\n" /* import barrier */ 175 "2:\t\n" /* oldval is zero if we set */ 176 : "=&r"(oldval) 177 : "r"(addr), "r"(temp) 178 : "cr0","memory"); 179 return oldval; 180 } 181 # define GC_TEST_AND_SET_DEFINED GC_clear(volatile unsigned int * addr)182 inline static void GC_clear(volatile unsigned int *addr) { 183 __asm__ __volatile__("lwsync" : : : "memory"); 184 *(addr) = 0; 185 } 186 # define GC_CLEAR_DEFINED 187 # endif 188 # if defined(ALPHA) GC_test_and_set(volatile unsigned int * addr)189 inline static int GC_test_and_set(volatile unsigned int * addr) 190 { 191 unsigned long oldvalue; 192 unsigned long temp; 193 194 __asm__ __volatile__( 195 "1: ldl_l %0,%1\n" 196 " and %0,%3,%2\n" 197 " bne %2,2f\n" 198 " xor %0,%3,%0\n" 199 " stl_c %0,%1\n" 200 # ifdef __ELF__ 201 " beq %0,3f\n" 202 # else 203 " beq %0,1b\n" 204 # endif 205 " mb\n" 206 "2:\n" 207 # ifdef __ELF__ 208 ".section .text2,\"ax\"\n" 209 "3: br 1b\n" 210 ".previous" 211 # endif 212 :"=&r" (temp), "=m" (*addr), "=&r" (oldvalue) 213 :"Ir" (1), "m" (*addr) 214 :"memory"); 215 216 return oldvalue; 217 } 218 # define GC_TEST_AND_SET_DEFINED GC_clear(volatile unsigned int * addr)219 inline static void GC_clear(volatile unsigned int *addr) { 220 __asm__ __volatile__("mb" : : : "memory"); 221 *(addr) = 0; 222 } 223 # define GC_CLEAR_DEFINED 224 # endif /* ALPHA */ 225 # ifdef ARM32 226 # ifdef __native_client__ 227 # define MASK_REGISTER(reg, cond) "bic" cond " " reg ", " reg ", #0xc0000000\n" 228 # define NACL_ALIGN() ".align 4\n" 229 # else 230 # define MASK_REGISTER(reg, cond) 231 # define NACL_ALIGN() 232 # endif GC_test_and_set(volatile unsigned int * addr)233 inline static int GC_test_and_set(volatile unsigned int *addr) { 234 return (int) __sync_lock_test_and_set (addr, 1); 235 } 236 # define GC_TEST_AND_SET_DEFINED GC_clear(volatile unsigned int * addr)237 inline static void GC_clear(volatile unsigned int *addr) { 238 __sync_synchronize (); 239 240 *(addr) = 0; 241 } 242 # define GC_CLEAR_DEFINED 243 # undef NACL_ALIGN 244 # undef MASK_REGISTER 245 # endif /* ARM32 */ 246 # ifdef CRIS GC_test_and_set(volatile unsigned int * addr)247 inline static int GC_test_and_set(volatile unsigned int *addr) { 248 /* Ripped from linuxthreads/sysdeps/cris/pt-machine.h. */ 249 /* Included with Hans-Peter Nilsson's permission. */ 250 register unsigned long int ret; 251 252 /* Note the use of a dummy output of *addr to expose the write. 253 * The memory barrier is to stop *other* writes being moved past 254 * this code. 255 */ 256 __asm__ __volatile__("clearf\n" 257 "0:\n\t" 258 "movu.b [%2],%0\n\t" 259 "ax\n\t" 260 "move.b %3,[%2]\n\t" 261 "bwf 0b\n\t" 262 "clearf" 263 : "=&r" (ret), "=m" (*addr) 264 : "r" (addr), "r" ((int) 1), "m" (*addr) 265 : "memory"); 266 return ret; 267 } 268 # define GC_TEST_AND_SET_DEFINED 269 # endif /* CRIS */ 270 # ifdef S390 GC_test_and_set(volatile unsigned int * addr)271 inline static int GC_test_and_set(volatile unsigned int *addr) { 272 int ret; 273 __asm__ __volatile__ ( 274 " l %0,0(%2)\n" 275 "0: cs %0,%1,0(%2)\n" 276 " jl 0b" 277 : "=&d" (ret) 278 : "d" (1), "a" (addr) 279 : "cc", "memory"); 280 return ret; 281 } 282 # endif 283 # endif /* __GNUC__ */ 284 # if (defined(ALPHA) && !defined(__GNUC__)) 285 # ifndef OSF1 286 --> We currently assume that if gcc is not used, we are 287 --> running under Tru64. 288 # endif 289 # include <machine/builtins.h> 290 # include <c_asm.h> 291 # define GC_test_and_set(addr) __ATOMIC_EXCH_LONG(addr, 1) 292 # define GC_TEST_AND_SET_DEFINED 293 # define GC_clear(addr) { asm("mb"); *(volatile unsigned *)addr = 0; } 294 # define GC_CLEAR_DEFINED 295 # endif 296 # if defined(MSWIN32) 297 # define GC_test_and_set(addr) InterlockedExchange((LPLONG)addr,1) 298 # define GC_TEST_AND_SET_DEFINED 299 # endif 300 # ifdef MIPS 301 # ifdef LINUX 302 # include <sys/tas.h> 303 # define GC_test_and_set(addr) _test_and_set((int *) addr,1) 304 # define GC_TEST_AND_SET_DEFINED 305 # elif __mips < 3 || !(defined (_ABIN32) || defined(_ABI64)) \ 306 || !defined(_COMPILER_VERSION) || _COMPILER_VERSION < 700 307 # ifdef __GNUC__ 308 # define GC_test_and_set(addr) _test_and_set((void *)addr,1) 309 # else 310 # define GC_test_and_set(addr) test_and_set((void *)addr,1) 311 # endif 312 # else 313 # include <sgidefs.h> 314 # include <mutex.h> 315 # define GC_test_and_set(addr) __test_and_set32((void *)addr,1) 316 # define GC_clear(addr) __lock_release(addr); 317 # define GC_CLEAR_DEFINED 318 # endif 319 # define GC_TEST_AND_SET_DEFINED 320 # endif /* MIPS */ 321 # if defined(_AIX) 322 # include <sys/atomic_op.h> 323 # if (defined(_POWER) || defined(_POWERPC)) 324 # if defined(__GNUC__) 325 inline static void GC_memsync() { 326 __asm__ __volatile__ ("sync" : : : "memory"); 327 } 328 # else 329 # ifndef inline 330 # define inline __inline 331 # endif 332 # pragma mc_func GC_memsync { \ 333 "7c0004ac" /* sync (same opcode used for dcs)*/ \ 334 } 335 # endif 336 # else 337 # error dont know how to memsync 338 # endif GC_test_and_set(volatile unsigned int * addr)339 inline static int GC_test_and_set(volatile unsigned int * addr) { 340 int oldvalue = 0; 341 if (compare_and_swap((void *)addr, &oldvalue, 1)) { 342 GC_memsync(); 343 return 0; 344 } else return 1; 345 } 346 # define GC_TEST_AND_SET_DEFINED GC_clear(volatile unsigned int * addr)347 inline static void GC_clear(volatile unsigned int *addr) { 348 GC_memsync(); 349 *(addr) = 0; 350 } 351 # define GC_CLEAR_DEFINED 352 353 # endif 354 # if 0 /* defined(HP_PA) */ 355 /* The official recommendation seems to be to not use ldcw from */ 356 /* user mode. Since multithreaded incremental collection doesn't */ 357 /* work anyway on HP_PA, this shouldn't be a major loss. */ 358 359 /* "set" means 0 and "clear" means 1 here. */ 360 # define GC_test_and_set(addr) !GC_test_and_clear(addr); 361 # define GC_TEST_AND_SET_DEFINED 362 # define GC_clear(addr) GC_noop1((word)(addr)); *(volatile unsigned int *)addr = 1; 363 /* The above needs a memory barrier! */ 364 # define GC_CLEAR_DEFINED 365 # endif 366 # if defined(GC_TEST_AND_SET_DEFINED) && !defined(GC_CLEAR_DEFINED) 367 # ifdef __GNUC__ GC_clear(volatile unsigned int * addr)368 inline static void GC_clear(volatile unsigned int *addr) { 369 /* Try to discourage gcc from moving anything past this. */ 370 __asm__ __volatile__(" " : : : "memory"); 371 *(addr) = 0; 372 } 373 # else 374 /* The function call in the following should prevent the */ 375 /* compiler from moving assignments to below the UNLOCK. */ 376 # define GC_clear(addr) GC_noop1((word)(addr)); \ 377 *((volatile unsigned int *)(addr)) = 0; 378 # endif 379 # define GC_CLEAR_DEFINED 380 # endif /* !GC_CLEAR_DEFINED */ 381 382 # if !defined(GC_TEST_AND_SET_DEFINED) 383 # define USE_PTHREAD_LOCKS 384 # endif 385 386 # if defined(GC_PTHREADS) && !defined(GC_SOLARIS_THREADS) \ 387 && !defined(GC_WIN32_THREADS) 388 # define NO_THREAD (pthread_t)(-1) 389 # include <pthread.h> 390 # if defined(PARALLEL_MARK) 391 /* We need compare-and-swap to update mark bits, where it's */ 392 /* performance critical. If USE_MARK_BYTES is defined, it is */ 393 /* no longer needed for this purpose. However we use it in */ 394 /* either case to implement atomic fetch-and-add, though that's */ 395 /* less performance critical, and could perhaps be done with */ 396 /* a lock. */ 397 # if defined(GENERIC_COMPARE_AND_SWAP) 398 /* Probably not useful, except for debugging. */ 399 /* We do use GENERIC_COMPARE_AND_SWAP on PA_RISC, but we */ 400 /* minimize its use. */ 401 extern pthread_mutex_t GC_compare_and_swap_lock; 402 403 /* Note that if GC_word updates are not atomic, a concurrent */ 404 /* reader should acquire GC_compare_and_swap_lock. On */ 405 /* currently supported platforms, such updates are atomic. */ 406 extern GC_bool GC_compare_and_exchange(volatile GC_word *addr, 407 GC_word old, GC_word new_val); 408 # endif /* GENERIC_COMPARE_AND_SWAP */ 409 # if defined(I386) 410 # if !defined(GENERIC_COMPARE_AND_SWAP) 411 /* Returns TRUE if the comparison succeeded. */ GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)412 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 413 GC_word old, 414 GC_word new_val) 415 { 416 char result; 417 __asm__ __volatile__("lock; cmpxchgl %2, %0; setz %1" 418 : "+m"(*(addr)), "=q"(result) 419 : "r" (new_val), "a"(old) : "memory"); 420 return (GC_bool) result; 421 } 422 # endif /* !GENERIC_COMPARE_AND_SWAP */ GC_memory_barrier()423 inline static void GC_memory_barrier() 424 { 425 /* We believe the processor ensures at least processor */ 426 /* consistent ordering. Thus a compiler barrier */ 427 /* should suffice. */ 428 __asm__ __volatile__("" : : : "memory"); 429 } 430 # endif /* I386 */ 431 432 # if defined(X86_64) 433 # if !defined(GENERIC_COMPARE_AND_SWAP) 434 /* Returns TRUE if the comparison succeeded. */ GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)435 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 436 GC_word old, 437 GC_word new_val) 438 { 439 char result; 440 __asm__ __volatile__("lock; cmpxchgq %2, %0; setz %1" 441 : "+m"(*(addr)), "=r"(result) 442 : "r" (new_val), "a"(old) : "memory"); 443 return (GC_bool) result; 444 } 445 # endif /* !GENERIC_COMPARE_AND_SWAP */ GC_memory_barrier()446 inline static void GC_memory_barrier() 447 { 448 /* We believe the processor ensures at least processor */ 449 /* consistent ordering. Thus a compiler barrier */ 450 /* should suffice. */ 451 __asm__ __volatile__("" : : : "memory"); 452 } 453 # endif /* X86_64 */ 454 455 # if defined(POWERPC) 456 # if !defined(GENERIC_COMPARE_AND_SWAP) 457 # if CPP_WORDSZ == 64 458 /* Returns TRUE if the comparison succeeded. */ GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)459 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 460 GC_word old, GC_word new_val) 461 { 462 # if HAS___SYNC_BOOL_COMPARE_AND_SWAP 463 return __sync_bool_compare_and_swap(addr, old, new_val); 464 # else 465 unsigned long result, dummy; 466 __asm__ __volatile__( 467 "1:\tldarx %0,0,%5\n" 468 "\tcmpd %0,%4\n" 469 "\tbne 2f\n" 470 "\tstdcx. %3,0,%2\n" 471 "\tbne- 1b\n" 472 "\tsync\n" 473 "\tli %1, 1\n" 474 "\tb 3f\n" 475 "2:\tli %1, 0\n" 476 "3:\t\n" 477 : "=&r" (dummy), "=r" (result), "=p" (addr) 478 : "r" (new_val), "r" (old), "2"(addr) 479 : "cr0","memory"); 480 return (GC_bool) result; 481 # endif 482 } 483 # else 484 /* Returns TRUE if the comparison succeeded. */ GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)485 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 486 GC_word old, GC_word new_val) 487 { 488 # if HAS___SYNC_BOOL_COMPARE_AND_SWAP 489 return __sync_bool_compare_and_swap(addr, old, new_val); 490 # else 491 int result, dummy; 492 __asm__ __volatile__( 493 "1:\tlwarx %0,0,%5\n" 494 "\tcmpw %0,%4\n" 495 "\tbne 2f\n" 496 "\tstwcx. %3,0,%2\n" 497 "\tbne- 1b\n" 498 "\tsync\n" 499 "\tli %1, 1\n" 500 "\tb 3f\n" 501 "2:\tli %1, 0\n" 502 "3:\t\n" 503 : "=&r" (dummy), "=r" (result), "=p" (addr) 504 : "r" (new_val), "r" (old), "2"(addr) 505 : "cr0","memory"); 506 return (GC_bool) result; 507 # endif 508 } 509 # endif 510 # endif /* !GENERIC_COMPARE_AND_SWAP */ GC_memory_barrier()511 inline static void GC_memory_barrier() 512 { 513 __asm__ __volatile__("sync" : : : "memory"); 514 } 515 # endif /* POWERPC */ 516 517 # if defined(SPARC) 518 # if !defined(GENERIC_COMPARE_AND_SWAP) 519 # if CPP_WORDSZ == 64 520 /* Returns TRUE if the comparison succeeded. */ GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)521 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 522 GC_word old, GC_word new_val) 523 { 524 unsigned long result; 525 __asm__ __volatile__( 526 "casx [%2], %3, %0" 527 : "=r" (result) 528 : "0" (new_val), "r" (addr), "r" (old) 529 : "memory"); 530 return (GC_bool) (result == old); 531 } 532 # else 533 /* Returns TRUE if the comparison succeeded. */ GC_compare_and_exchange(volatile GC_word * _addr,GC_word _old,GC_word _new_val)534 inline static GC_bool GC_compare_and_exchange(volatile GC_word *_addr, 535 GC_word _old, GC_word _new_val) 536 { 537 register unsigned long result asm("o0"); 538 register unsigned long old asm("o1"); 539 register volatile GC_word *addr asm("o2"); 540 result = _new_val; 541 old = _old; 542 addr = _addr; 543 __asm__ __volatile__( 544 /* We encode the instruction directly so that it 545 doesn't taint the whole binary as v9-only. */ 546 ".word 0xd1e29009" /* cas [%o2], %o1, %o0 */ 547 : "=r" (result) 548 : "0" (result), "r" (addr), "r"(old) 549 : "memory"); 550 return (GC_bool) (result == old); 551 } 552 # endif 553 # endif /* !GENERIC_COMPARE_AND_SWAP */ GC_memory_barrier()554 inline static void GC_memory_barrier() 555 { 556 /* All sparc v9 chips provice procesor consistent ordering. */ 557 /* Thus a compiler barrier should suffice. */ 558 __asm__ __volatile__("" : : : "memory"); 559 } 560 # endif /* SPARC */ 561 562 # if defined(IA64) 563 # if !defined(GENERIC_COMPARE_AND_SWAP) GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)564 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 565 GC_word old, GC_word new_val) 566 { 567 unsigned long oldval; 568 # if CPP_WORDSZ == 32 569 __asm__ __volatile__( 570 "addp4 %0=0,%1\n" 571 "mov ar.ccv=%3 ;; cmpxchg4.rel %0=[%0],%2,ar.ccv" 572 : "=&r"(oldval) 573 : "r"(addr), "r"(new_val), "r"(old) : "memory"); 574 # else 575 __asm__ __volatile__( 576 "mov ar.ccv=%3 ;; cmpxchg8.rel %0=[%1],%2,ar.ccv" 577 : "=r"(oldval) 578 : "r"(addr), "r"(new_val), "r"(old) : "memory"); 579 # endif 580 return (oldval == old); 581 } 582 # endif /* !GENERIC_COMPARE_AND_SWAP */ 583 # if 0 584 /* Shouldn't be needed; we use volatile stores instead. */ 585 inline static void GC_memory_barrier() 586 { 587 __asm__ __volatile__("mf" : : : "memory"); 588 } 589 # endif /* 0 */ 590 # endif /* IA64 */ 591 # if defined(ALPHA) 592 # if !defined(GENERIC_COMPARE_AND_SWAP) 593 # if defined(__GNUC__) GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)594 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 595 GC_word old, GC_word new_val) 596 { 597 unsigned long was_equal; 598 unsigned long temp; 599 600 __asm__ __volatile__( 601 "1: ldq_l %0,%1\n" 602 " cmpeq %0,%4,%2\n" 603 " mov %3,%0\n" 604 " beq %2,2f\n" 605 " stq_c %0,%1\n" 606 " beq %0,1b\n" 607 "2:\n" 608 " mb\n" 609 :"=&r" (temp), "=m" (*addr), "=&r" (was_equal) 610 : "r" (new_val), "Ir" (old) 611 :"memory"); 612 return was_equal; 613 } 614 # else /* !__GNUC__ */ GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)615 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 616 GC_word old, GC_word new_val) 617 { 618 return __CMP_STORE_QUAD(addr, old, new_val, addr); 619 } 620 # endif /* !__GNUC__ */ 621 # endif /* !GENERIC_COMPARE_AND_SWAP */ 622 # ifdef __GNUC__ GC_memory_barrier()623 inline static void GC_memory_barrier() 624 { 625 __asm__ __volatile__("mb" : : : "memory"); 626 } 627 # else 628 # define GC_memory_barrier() asm("mb") 629 # endif /* !__GNUC__ */ 630 # endif /* ALPHA */ 631 # if defined(S390) 632 # if !defined(GENERIC_COMPARE_AND_SWAP) GC_compare_and_exchange(volatile GC_word * addr,GC_word old,GC_word new_val)633 inline static GC_bool GC_compare_and_exchange(volatile GC_word *addr, 634 GC_word old, GC_word new_val) 635 { 636 int retval; 637 __asm__ __volatile__ ( 638 # ifndef __s390x__ 639 " cs %1,%2,0(%3)\n" 640 # else 641 " csg %1,%2,0(%3)\n" 642 # endif 643 " ipm %0\n" 644 " srl %0,28\n" 645 : "=&d" (retval), "+d" (old) 646 : "d" (new_val), "a" (addr) 647 : "cc", "memory"); 648 return retval == 0; 649 } 650 # endif 651 # define GC_memory_barrier() 652 # endif 653 # if !defined(GENERIC_COMPARE_AND_SWAP) 654 /* Returns the original value of *addr. */ GC_atomic_add(volatile GC_word * addr,GC_word how_much)655 inline static GC_word GC_atomic_add(volatile GC_word *addr, 656 GC_word how_much) 657 { 658 GC_word old; 659 do { 660 old = *addr; 661 } while (!GC_compare_and_exchange(addr, old, old+how_much)); 662 return old; 663 } 664 # else /* GENERIC_COMPARE_AND_SWAP */ 665 /* So long as a GC_word can be atomically updated, it should */ 666 /* be OK to read *addr without a lock. */ 667 extern GC_word GC_atomic_add(volatile GC_word *addr, GC_word how_much); 668 # endif /* GENERIC_COMPARE_AND_SWAP */ 669 670 # endif /* PARALLEL_MARK */ 671 672 # if !defined(THREAD_LOCAL_ALLOC) && !defined(USE_PTHREAD_LOCKS) 673 /* In the THREAD_LOCAL_ALLOC case, the allocation lock tends to */ 674 /* be held for long periods, if it is held at all. Thus spinning */ 675 /* and sleeping for fixed periods are likely to result in */ 676 /* significant wasted time. We thus rely mostly on queued locks. */ 677 # define USE_SPIN_LOCK 678 extern volatile unsigned int GC_allocate_lock; 679 extern void GC_lock(void); 680 /* Allocation lock holder. Only set if acquired by client through */ 681 /* GC_call_with_alloc_lock. */ 682 # ifdef GC_ASSERTIONS 683 # define LOCK() \ 684 { if (GC_test_and_set(&GC_allocate_lock)) GC_lock(); \ 685 SET_LOCK_HOLDER(); } 686 # define UNLOCK() \ 687 { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \ 688 GC_clear(&GC_allocate_lock); } 689 # else 690 # define LOCK() \ 691 { if (GC_test_and_set(&GC_allocate_lock)) GC_lock(); } 692 # define UNLOCK() \ 693 GC_clear(&GC_allocate_lock) 694 # endif /* !GC_ASSERTIONS */ 695 # if 0 696 /* Another alternative for OSF1 might be: */ 697 # include <sys/mman.h> 698 extern msemaphore GC_allocate_semaphore; 699 # define LOCK() { if (msem_lock(&GC_allocate_semaphore, MSEM_IF_NOWAIT) \ 700 != 0) GC_lock(); else GC_allocate_lock = 1; } 701 /* The following is INCORRECT, since the memory model is too weak. */ 702 /* Is this true? Presumably msem_unlock has the right semantics? */ 703 /* - HB */ 704 # define UNLOCK() { GC_allocate_lock = 0; \ 705 msem_unlock(&GC_allocate_semaphore, 0); } 706 # endif /* 0 */ 707 # else /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */ 708 # ifndef USE_PTHREAD_LOCKS 709 # define USE_PTHREAD_LOCKS 710 # endif 711 # endif /* THREAD_LOCAL_ALLOC */ 712 # ifdef USE_PTHREAD_LOCKS 713 # include <pthread.h> 714 extern pthread_mutex_t GC_allocate_ml; 715 # ifdef GC_ASSERTIONS 716 # define LOCK() \ 717 { GC_lock(); \ 718 SET_LOCK_HOLDER(); } 719 # define UNLOCK() \ 720 { GC_ASSERT(I_HOLD_LOCK()); UNSET_LOCK_HOLDER(); \ 721 pthread_mutex_unlock(&GC_allocate_ml); } 722 # else /* !GC_ASSERTIONS */ 723 # if defined(NO_PTHREAD_TRYLOCK) 724 # define LOCK() GC_lock(); 725 # else /* !defined(NO_PTHREAD_TRYLOCK) */ 726 # define LOCK() \ 727 { if (0 != pthread_mutex_trylock(&GC_allocate_ml)) GC_lock(); } 728 # endif 729 # define UNLOCK() pthread_mutex_unlock(&GC_allocate_ml) 730 # endif /* !GC_ASSERTIONS */ 731 # endif /* USE_PTHREAD_LOCKS */ 732 # define SET_LOCK_HOLDER() GC_lock_holder = pthread_self() 733 # define UNSET_LOCK_HOLDER() GC_lock_holder = NO_THREAD 734 # define I_HOLD_LOCK() (pthread_equal(GC_lock_holder, pthread_self())) 735 extern VOLATILE GC_bool GC_collecting; 736 # define ENTER_GC() GC_collecting = 1; 737 # define EXIT_GC() GC_collecting = 0; 738 extern void GC_lock(void); 739 extern pthread_t GC_lock_holder; 740 # ifdef GC_ASSERTIONS 741 extern pthread_t GC_mark_lock_holder; 742 # endif 743 # endif /* GC_PTHREADS with linux_threads.c implementation */ 744 # if defined(GC_WIN32_THREADS) 745 # if defined(GC_PTHREADS) 746 # include <pthread.h> 747 extern pthread_mutex_t GC_allocate_ml; 748 # define LOCK() pthread_mutex_lock(&GC_allocate_ml) 749 # define UNLOCK() pthread_mutex_unlock(&GC_allocate_ml) 750 # else 751 # include <windows.h> 752 GC_API CRITICAL_SECTION GC_allocate_ml; 753 # define LOCK() EnterCriticalSection(&GC_allocate_ml); 754 # define UNLOCK() LeaveCriticalSection(&GC_allocate_ml); 755 # endif 756 # endif 757 # ifndef SET_LOCK_HOLDER 758 # define SET_LOCK_HOLDER() 759 # define UNSET_LOCK_HOLDER() 760 # define I_HOLD_LOCK() FALSE 761 /* Used on platforms were locks can be reacquired, */ 762 /* so it doesn't matter if we lie. */ 763 # endif 764 # else /* !THREADS */ 765 # define LOCK() 766 # define UNLOCK() 767 # endif /* !THREADS */ 768 # ifndef SET_LOCK_HOLDER 769 # define SET_LOCK_HOLDER() 770 # define UNSET_LOCK_HOLDER() 771 # define I_HOLD_LOCK() FALSE 772 /* Used on platforms were locks can be reacquired, */ 773 /* so it doesn't matter if we lie. */ 774 # endif 775 # ifndef ENTER_GC 776 # define ENTER_GC() 777 # define EXIT_GC() 778 # endif 779 780 # ifndef DCL_LOCK_STATE 781 # define DCL_LOCK_STATE 782 # endif 783 # ifndef FASTLOCK 784 # define FASTLOCK() LOCK() 785 # define FASTLOCK_SUCCEEDED() TRUE 786 # define FASTUNLOCK() UNLOCK() 787 # endif 788 789 #endif /* GC_LOCKS_H */ 790