1/***************************************************************************** 2 3Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. 4Copyright (c) 2008, Google Inc. 5 6Portions of this file contain modifications contributed and copyrighted by 7Google, Inc. Those modifications are gratefully acknowledged and are described 8briefly in the InnoDB documentation. The contributions by Google are 9incorporated with their permission, and subject to the conditions contained in 10the file COPYING.Google. 11 12This program is free software; you can redistribute it and/or modify 13it under the terms of the GNU General Public License, version 2.0, 14as published by the Free Software Foundation. 15 16This program is also distributed with certain software (including 17but not limited to OpenSSL) that is licensed under separate terms, 18as designated in a particular file or component or in included license 19documentation. The authors of MySQL hereby grant you an additional 20permission to link the program and your derivative works with the 21separately licensed software that they have included with MySQL. 22 23This program is distributed in the hope that it will be useful, 24but WITHOUT ANY WARRANTY; without even the implied warranty of 25MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 26GNU General Public License, version 2.0, for more details. 27 28You should have received a copy of the GNU General Public License along with 29this program; if not, write to the Free Software Foundation, Inc., 3051 Franklin Street, Suite 500, Boston, MA 02110-1335 USA 31 32*****************************************************************************/ 33 34/**************************************************//** 35@file include/sync0sync.ic 36Mutex, the basic synchronization primitive 37 38Created 9/5/1995 Heikki Tuuri 39*******************************************************/ 40 41/******************************************************************//** 42Sets the waiters field in a mutex. */ 43UNIV_INTERN 44void 45mutex_set_waiters( 46/*==============*/ 47 ib_mutex_t* mutex, /*!< in: mutex */ 48 ulint n); /*!< in: value to set */ 49/******************************************************************//** 50Reserves a mutex or a priority mutex for the current thread. If the mutex is 51reserved, the function spins a preset time (controlled by SYNC_SPIN_ROUNDS) 52waiting for the mutex before suspending the thread. */ 53UNIV_INTERN 54void 55mutex_spin_wait( 56/*============*/ 57 void* _mutex, /*!< in: pointer to mutex */ 58 bool high_priority, /*!< in: whether the mutex is a 59 priority mutex with high priority 60 specified */ 61 const char* file_name, /*!< in: file name where mutex 62 requested */ 63 ulint line); /*!< in: line where requested */ 64#ifdef UNIV_SYNC_DEBUG 65/******************************************************************//** 66Sets the debug information for a reserved mutex. */ 67UNIV_INTERN 68void 69mutex_set_debug_info( 70/*=================*/ 71 ib_mutex_t* mutex, /*!< in: mutex */ 72 const char* file_name, /*!< in: file where requested */ 73 ulint line); /*!< in: line where requested */ 74#endif /* UNIV_SYNC_DEBUG */ 75/******************************************************************//** 76Releases the threads waiting in the primary wait array for this mutex. */ 77UNIV_INTERN 78void 79mutex_signal_object( 80/*================*/ 81 ib_mutex_t* mutex); /*!< in: mutex */ 82 83/******************************************************************//** 84Performs an atomic test-and-set instruction to the lock_word field of a 85mutex. 86@return the previous value of lock_word: 0 or 1 */ 87UNIV_INLINE 88lock_word_t 89ib_mutex_test_and_set( 90/*==================*/ 91 ib_mutex_t* mutex) /*!< in: mutex */ 92{ 93#if defined(HAVE_ATOMIC_BUILTINS) 94 return(os_atomic_test_and_set(&mutex->lock_word)); 95#else 96 ibool ret; 97 98 ret = os_fast_mutex_trylock(&(mutex->os_fast_mutex)); 99 100 if (ret == 0) { 101 /* We check that os_fast_mutex_trylock does not leak 102 and allow race conditions */ 103 ut_a(mutex->lock_word == 0); 104 105 mutex->lock_word = 1; 106 os_wmb; 107 } 108 109 return((byte) ret); 110#endif /* HAVE_ATOMIC_BUILTINS */ 111} 112 113/******************************************************************//** 114Performs a reset instruction to the lock_word field of a mutex. This 115instruction also serializes memory operations to the program order. */ 116UNIV_INLINE 117void 118mutex_reset_lock_word( 119/*==================*/ 120 ib_mutex_t* mutex) /*!< in: mutex */ 121{ 122#if defined(HAVE_ATOMIC_BUILTINS) 123 os_atomic_clear(&mutex->lock_word); 124#else 125 mutex->lock_word = 0; 126 127 os_fast_mutex_unlock(&(mutex->os_fast_mutex)); 128#endif /* HAVE_ATOMIC_BUILTINS */ 129} 130 131/******************************************************************//** 132Gets the value of the lock word. */ 133UNIV_INLINE 134lock_word_t 135mutex_get_lock_word( 136/*================*/ 137 const ib_mutex_t* mutex) /*!< in: mutex */ 138{ 139 ut_ad(mutex); 140 141 return(mutex->lock_word); 142} 143 144/******************************************************************//** 145Gets the waiters field in a mutex. 146@return value to set */ 147UNIV_INLINE 148ulint 149mutex_get_waiters( 150/*==============*/ 151 const ib_mutex_t* mutex) /*!< in: mutex */ 152{ 153 const volatile ulint* ptr; /*!< declared volatile to ensure that 154 the value is read from memory */ 155 ut_ad(mutex); 156 157 ptr = &(mutex->waiters); 158 159 return(*ptr); /* Here we assume that the read of a single 160 word from memory is atomic */ 161} 162 163/******************************************************************//** 164NOTE! Use the corresponding macro mutex_exit(), not directly this function! 165Unlocks a mutex owned by the current thread. */ 166UNIV_INLINE 167void 168mutex_exit_func( 169/*============*/ 170 ib_mutex_t* mutex) /*!< in: pointer to mutex */ 171{ 172 ut_ad(mutex_own(mutex)); 173 174 ut_d(mutex->thread_id = (os_thread_id_t) ULINT_UNDEFINED); 175 176#ifdef UNIV_SYNC_DEBUG 177 sync_thread_reset_level(mutex); 178#endif 179 mutex_reset_lock_word(mutex); 180 181 /* A problem: we assume that mutex_reset_lock word 182 is a memory barrier, that is when we read the waiters 183 field next, the read must be serialized in memory 184 after the reset. A speculative processor might 185 perform the read first, which could leave a waiting 186 thread hanging indefinitely. 187 188 Our current solution call every second 189 sync_arr_wake_threads_if_sema_free() 190 to wake up possible hanging threads if 191 they are missed in mutex_signal_object. */ 192 193 if (mutex_get_waiters(mutex) != 0) { 194 195 mutex_signal_object(mutex); 196 } 197 198#ifdef UNIV_SYNC_PERF_STAT 199 mutex_exit_count++; 200#endif 201} 202 203/******************************************************************//** 204NOTE! Use the corresponding macro mutex_exit(), not directly this function! 205Unlocks a priority mutex owned by the current thread. */ 206UNIV_INLINE 207void 208mutex_exit_func( 209/*============*/ 210 ib_prio_mutex_t* mutex) /*!< in: pointer to mutex */ 211{ 212 ut_ad(mutex_own(mutex)); 213 214 ut_d(mutex->base_mutex.thread_id = (os_thread_id_t) ULINT_UNDEFINED); 215 216#ifdef UNIV_SYNC_DEBUG 217 sync_thread_reset_level(&mutex->base_mutex); 218#endif 219 mutex_reset_lock_word(&mutex->base_mutex); 220 221 /* A problem: we assume that mutex_reset_lock word 222 is a memory barrier, that is when we read the waiters 223 field next, the read must be serialized in memory 224 after the reset. A speculative processor might 225 perform the read first, which could leave a waiting 226 thread hanging indefinitely. 227 228 Our current solution call every second 229 sync_arr_wake_threads_if_sema_free() 230 to wake up possible hanging threads if 231 they are missed in mutex_signal_object. */ 232 233 /* Wake up any high priority waiters first. */ 234 if (mutex->high_priority_waiters != 0) { 235 236 os_event_set(&mutex->high_priority_event); 237 sync_array_object_signalled(); 238 239 } else if (mutex_get_waiters(&mutex->base_mutex) != 0) { 240 241 mutex_signal_object(&mutex->base_mutex); 242 } 243 244#ifdef UNIV_SYNC_PERF_STAT 245 mutex_exit_count++; 246#endif 247 248} 249 250 251/******************************************************************//** 252Locks a mutex for the current thread. If the mutex is reserved, the function 253spins a preset time (controlled by SYNC_SPIN_ROUNDS), waiting for the mutex 254before suspending the thread. */ 255UNIV_INLINE 256void 257mutex_enter_func( 258/*=============*/ 259 ib_mutex_t* mutex, /*!< in: pointer to mutex */ 260 const char* file_name, /*!< in: file name where locked */ 261 ulint line) /*!< in: line where locked */ 262{ 263 ut_ad(mutex_validate(mutex)); 264 ut_ad(!mutex_own(mutex)); 265 266 /* Note that we do not peek at the value of lock_word before trying 267 the atomic test_and_set; we could peek, and possibly save time. */ 268 269 if (!ib_mutex_test_and_set(mutex)) { 270 ut_d(mutex->thread_id = os_thread_get_curr_id()); 271#ifdef UNIV_SYNC_DEBUG 272 mutex_set_debug_info(mutex, file_name, line); 273#endif 274 return; /* Succeeded! */ 275 } 276 277 mutex_spin_wait(mutex, false, file_name, line); 278} 279 280/******************************************************************//** 281NOTE! Use the corresponding macro in the header file, not this function 282directly. Locks a priority mutex for the current thread. If the mutex is 283reserved the function spins a preset time (controlled by SYNC_SPIN_ROUNDS) 284waiting for the mutex before suspending the thread. If the thread is suspended, 285the priority argument value determines the relative order for its wake up. Any 286HIGH_PRIO waiters will be woken up before any LOW_PRIO waiters. In case of 287DEFAULT_PRIO, the relative priority will be set according to 288srv_current_thread_priority. */ 289UNIV_INLINE 290void 291mutex_enter_func( 292/*=============*/ 293 ib_prio_mutex_t* mutex, /*!< in: pointer to mutex */ 294 const char* file_name, /*!< in: file name where 295 locked */ 296 ulint line, /*!< in: line where locked */ 297 enum ib_sync_priority priority) 298 /*!<in: mutex acquisition 299 priority */ 300{ 301 bool high_priority; 302 303 ut_ad(mutex_validate(&mutex->base_mutex)); 304 ut_ad(!mutex_own(mutex)); 305 306 /* Note that we do not peek at the value of lock_word before trying 307 the atomic test_and_set; we could peek, and possibly save time. */ 308 309 if (!ib_mutex_test_and_set(&mutex->base_mutex)) { 310 ut_d(mutex->base_mutex.thread_id = os_thread_get_curr_id()); 311#ifdef UNIV_SYNC_DEBUG 312 mutex_set_debug_info(&mutex->base_mutex, file_name, line); 313#endif 314 return; /* Succeeded! */ 315 } 316 317 if (UNIV_LIKELY(priority == DEFAULT_PRIO)) { 318 high_priority = srv_current_thread_priority; 319 } else { 320 high_priority = (priority == HIGH_PRIO); 321 } 322 mutex_spin_wait(mutex, high_priority, file_name, line); 323} 324 325/********************************************************************//** 326NOTE! Use the corresponding macro in the header file, not this function 327directly. Tries to lock the mutex for the current thread. If the lock is not 328acquired immediately, returns with return value 1. 329@return 0 if succeed, 1 if not */ 330UNIV_INLINE 331ulint 332mutex_enter_nowait_func( 333/*====================*/ 334 ib_prio_mutex_t* mutex, /*!< in: pointer to mutex */ 335 const char* file_name, /*!< in: file name where mutex 336 requested */ 337 ulint line) /*!< in: line where 338 requested */ 339{ 340 return mutex_enter_nowait_func(&mutex->base_mutex, file_name, line); 341} 342 343#ifdef UNIV_PFS_MUTEX 344/******************************************************************//** 345NOTE! Please use the corresponding macro mutex_enter(), not directly 346this function! 347This is a performance schema instrumented wrapper function for 348mutex_enter_func(). */ 349UNIV_INLINE 350void 351pfs_mutex_enter_func( 352/*=================*/ 353 ib_mutex_t* mutex, /*!< in: pointer to mutex */ 354 const char* file_name, /*!< in: file name where locked */ 355 ulint line) /*!< in: line where locked */ 356{ 357 if (mutex->pfs_psi != NULL) { 358 PSI_mutex_locker* locker; 359 PSI_mutex_locker_state state; 360 361 locker = PSI_MUTEX_CALL(start_mutex_wait)( 362 &state, mutex->pfs_psi, 363 PSI_MUTEX_LOCK, file_name, 364 static_cast<uint>(line)); 365 366 mutex_enter_func(mutex, file_name, line); 367 368 if (locker != NULL) { 369 PSI_MUTEX_CALL(end_mutex_wait)(locker, 0); 370 } 371 } else { 372 mutex_enter_func(mutex, file_name, line); 373 } 374} 375 376/******************************************************************//** 377NOTE! Please use the corresponding macro mutex_enter(), not directly 378this function! 379This is a performance schema instrumented wrapper function for 380mutex_enter_func(). */ 381UNIV_INLINE 382void 383pfs_mutex_enter_func( 384/*=================*/ 385 ib_prio_mutex_t* mutex, /*!< in: pointer to mutex */ 386 const char* file_name, /*!< in: file name where 387 locked */ 388 ulint line, /*!< in: line where locked */ 389 enum ib_sync_priority priority) /*!<in: mutex acquisition 390 priority */ 391{ 392 if (mutex->base_mutex.pfs_psi != NULL) { 393 PSI_mutex_locker* locker; 394 PSI_mutex_locker_state state; 395 396 locker = PSI_MUTEX_CALL(start_mutex_wait)( 397 &state, mutex->base_mutex.pfs_psi, 398 PSI_MUTEX_LOCK, file_name, line); 399 400 mutex_enter_func(mutex, file_name, line, priority); 401 402 if (locker != NULL) { 403 PSI_MUTEX_CALL(end_mutex_wait)(locker, 0); 404 } 405 } else { 406 mutex_enter_func(mutex, file_name, line, priority); 407 } 408} 409 410/********************************************************************//** 411NOTE! Please use the corresponding macro mutex_enter_nowait(), not directly 412this function! 413This is a performance schema instrumented wrapper function for 414mutex_enter_nowait_func. 415@return 0 if succeed, 1 if not */ 416UNIV_INLINE 417ulint 418pfs_mutex_enter_nowait_func( 419/*========================*/ 420 ib_mutex_t* mutex, /*!< in: pointer to mutex */ 421 const char* file_name, /*!< in: file name where mutex 422 requested */ 423 ulint line) /*!< in: line where requested */ 424{ 425 ulint ret; 426 427 if (mutex->pfs_psi != NULL) { 428 PSI_mutex_locker* locker; 429 PSI_mutex_locker_state state; 430 431 locker = PSI_MUTEX_CALL(start_mutex_wait)( 432 &state, mutex->pfs_psi, 433 PSI_MUTEX_TRYLOCK, file_name, 434 static_cast<uint>(line)); 435 436 ret = mutex_enter_nowait_func(mutex, file_name, line); 437 438 if (locker != NULL) { 439 PSI_MUTEX_CALL(end_mutex_wait)(locker, (int) ret); 440 } 441 } else { 442 ret = mutex_enter_nowait_func(mutex, file_name, line); 443 } 444 445 return(ret); 446} 447 448/********************************************************************//** 449NOTE! Please use the corresponding macro mutex_enter_nowait(), not directly 450this function! 451This is a performance schema instrumented wrapper function for 452mutex_enter_nowait_func. 453@return 0 if succeed, 1 if not */ 454UNIV_INLINE 455ulint 456pfs_mutex_enter_nowait_func( 457/*========================*/ 458 ib_prio_mutex_t* mutex, /*!< in: pointer to mutex */ 459 const char* file_name, /*!< in: file name where mutex 460 requested */ 461 ulint line) /*!< in: line where 462 requested */ 463{ 464 return pfs_mutex_enter_nowait_func(&mutex->base_mutex, file_name, 465 line); 466} 467 468/******************************************************************//** 469NOTE! Please use the corresponding macro mutex_exit(), not directly 470this function! 471A wrap function of mutex_exit_func() with performance schema instrumentation. 472Unlocks a mutex owned by the current thread. */ 473UNIV_INLINE 474void 475pfs_mutex_exit_func( 476/*================*/ 477 ib_mutex_t* mutex) /*!< in: pointer to mutex */ 478{ 479 if (mutex->pfs_psi != NULL) { 480 PSI_MUTEX_CALL(unlock_mutex)(mutex->pfs_psi); 481 } 482 483 mutex_exit_func(mutex); 484} 485 486/******************************************************************//** 487NOTE! Please use the corresponding macro mutex_exit(), not directly 488this function! 489A wrap function of mutex_exit_func() with peformance schema instrumentation. 490Unlocks a priority mutex owned by the current thread. */ 491UNIV_INLINE 492void 493pfs_mutex_exit_func( 494/*================*/ 495 ib_prio_mutex_t* mutex) /*!< in: pointer to mutex */ 496{ 497 if (mutex->base_mutex.pfs_psi != NULL) { 498 PSI_MUTEX_CALL(unlock_mutex)(mutex->base_mutex.pfs_psi); 499 } 500 501 mutex_exit_func(mutex); 502} 503 504 505/******************************************************************//** 506NOTE! Please use the corresponding macro mutex_create(), not directly 507this function! 508A wrapper function for mutex_create_func(), registers the mutex 509with performance schema if "UNIV_PFS_MUTEX" is defined when 510creating the mutex */ 511UNIV_INLINE 512void 513pfs_mutex_create_func( 514/*==================*/ 515 mysql_pfs_key_t key, /*!< in: Performance Schema key */ 516 ib_mutex_t* mutex, /*!< in: pointer to memory */ 517# ifdef UNIV_DEBUG 518# ifdef UNIV_SYNC_DEBUG 519 ulint level, /*!< in: level */ 520# endif /* UNIV_SYNC_DEBUG */ 521 const char* cfile_name, /*!< in: file name where created */ 522 ulint cline, /*!< in: file line where created */ 523# endif /* UNIV_DEBUG */ 524 const char* cmutex_name) /*!< in: mutex name */ 525{ 526 mutex->pfs_psi = PSI_MUTEX_CALL(init_mutex)(key, mutex); 527 528 mutex_create_func(mutex, 529# ifdef UNIV_DEBUG 530# ifdef UNIV_SYNC_DEBUG 531 level, 532# endif /* UNIV_SYNC_DEBUG */ 533 cfile_name, 534 cline, 535# endif /* UNIV_DEBUG */ 536 cmutex_name); 537} 538 539/******************************************************************//** 540NOTE! Please use the corresponding macro mutex_create(), not directly 541this function! 542A wrapper function for mutex_create_func(), registers the mutex 543with peformance schema if "UNIV_PFS_MUTEX" is defined when 544creating the performance mutex */ 545UNIV_INLINE 546void 547pfs_mutex_create_func( 548/*==================*/ 549 PSI_mutex_key key, /*!< in: Performance Schema 550 key */ 551 ib_prio_mutex_t* mutex, /*!< in: pointer to memory */ 552# ifdef UNIV_DEBUG 553# ifdef UNIV_SYNC_DEBUG 554 ulint level, /*!< in: level */ 555# endif /* UNIV_SYNC_DEBUG */ 556 const char* cfile_name, /*!< in: file name where 557 created */ 558 ulint cline, /*!< in: file line where 559 created */ 560# endif /* UNIV_DEBUG */ 561 const char* cmutex_name) 562{ 563 mutex->base_mutex.pfs_psi = PSI_MUTEX_CALL(init_mutex)(key, mutex); 564 565 mutex_create_func(mutex, 566# ifdef UNIV_DEBUG 567# ifdef UNIV_SYNC_DEBUG 568 level, 569# endif /* UNIV_SYNC_DEBUG */ 570 cfile_name, 571 cline, 572# endif /* UNIV_DEBUG */ 573 cmutex_name); 574} 575 576 577/******************************************************************//** 578NOTE! Please use the corresponding macro mutex_free(), not directly 579this function! 580Wrapper function for mutex_free_func(). Also destroys the performance 581schema probes when freeing the mutex */ 582UNIV_INLINE 583void 584pfs_mutex_free_func( 585/*================*/ 586 ib_mutex_t* mutex) /*!< in: mutex */ 587{ 588 if (mutex->pfs_psi != NULL) { 589 PSI_MUTEX_CALL(destroy_mutex)(mutex->pfs_psi); 590 mutex->pfs_psi = NULL; 591 } 592 593 mutex_free_func(mutex); 594} 595 596/******************************************************************//** 597NOTE! Please use the corresponding macro mutex_free(), not directly 598this function! 599Wrapper function for mutex_free_func(). Also destroys the performance 600schema probes when freeing the priority mutex */ 601UNIV_INLINE 602void 603pfs_mutex_free_func( 604/*================*/ 605 ib_prio_mutex_t* mutex) /*!< in: mutex */ 606{ 607 if (mutex->base_mutex.pfs_psi != NULL) { 608 PSI_MUTEX_CALL(destroy_mutex)(mutex->base_mutex.pfs_psi); 609 mutex->base_mutex.pfs_psi = NULL; 610 } 611 612 mutex_free_func(mutex); 613} 614 615 616#endif /* UNIV_PFS_MUTEX */ 617 618#ifndef HAVE_ATOMIC_BUILTINS 619/**********************************************************//** 620Function that uses a mutex to decrement a variable atomically */ 621UNIV_INLINE 622void 623os_atomic_dec_ulint_func( 624/*=====================*/ 625 ib_mutex_t* mutex, /*!< in: mutex guarding the dec */ 626 volatile ulint* var, /*!< in/out: variable to decrement */ 627 ulint delta) /*!< in: delta to decrement */ 628{ 629 mutex_enter(mutex); 630 631 /* I don't think we will encounter a situation where 632 this check will not be required. */ 633 ut_ad(*var >= delta); 634 635 *var -= delta; 636 637 mutex_exit(mutex); 638} 639 640/**********************************************************//** 641Function that uses a mutex to increment a variable atomically */ 642UNIV_INLINE 643void 644os_atomic_inc_ulint_func( 645/*=====================*/ 646 ib_mutex_t* mutex, /*!< in: mutex guarding the increment */ 647 volatile ulint* var, /*!< in/out: variable to increment */ 648 ulint delta) /*!< in: delta to increment */ 649{ 650 mutex_enter(mutex); 651 652 *var += delta; 653 654 mutex_exit(mutex); 655} 656#endif /* !HAVE_ATOMIC_BUILTINS */ 657