1 /*
2 * z_Windows_NT_util.cpp -- platform specific routines.
3 */
4
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_wait_release.h"
19
20 /* This code is related to NtQuerySystemInformation() function. This function
21 is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
22 number of running threads in the system. */
23
24 #include <ntsecapi.h> // UNICODE_STRING
25 #include <ntstatus.h>
26
27 enum SYSTEM_INFORMATION_CLASS {
28 SystemProcessInformation = 5
29 }; // SYSTEM_INFORMATION_CLASS
30
31 struct CLIENT_ID {
32 HANDLE UniqueProcess;
33 HANDLE UniqueThread;
34 }; // struct CLIENT_ID
35
36 enum THREAD_STATE {
37 StateInitialized,
38 StateReady,
39 StateRunning,
40 StateStandby,
41 StateTerminated,
42 StateWait,
43 StateTransition,
44 StateUnknown
45 }; // enum THREAD_STATE
46
47 struct VM_COUNTERS {
48 SIZE_T PeakVirtualSize;
49 SIZE_T VirtualSize;
50 ULONG PageFaultCount;
51 SIZE_T PeakWorkingSetSize;
52 SIZE_T WorkingSetSize;
53 SIZE_T QuotaPeakPagedPoolUsage;
54 SIZE_T QuotaPagedPoolUsage;
55 SIZE_T QuotaPeakNonPagedPoolUsage;
56 SIZE_T QuotaNonPagedPoolUsage;
57 SIZE_T PagefileUsage;
58 SIZE_T PeakPagefileUsage;
59 SIZE_T PrivatePageCount;
60 }; // struct VM_COUNTERS
61
62 struct SYSTEM_THREAD {
63 LARGE_INTEGER KernelTime;
64 LARGE_INTEGER UserTime;
65 LARGE_INTEGER CreateTime;
66 ULONG WaitTime;
67 LPVOID StartAddress;
68 CLIENT_ID ClientId;
69 DWORD Priority;
70 LONG BasePriority;
71 ULONG ContextSwitchCount;
72 THREAD_STATE State;
73 ULONG WaitReason;
74 }; // SYSTEM_THREAD
75
76 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
77 #if KMP_ARCH_X86
78 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
80 #else
81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
83 #endif
84
85 struct SYSTEM_PROCESS_INFORMATION {
86 ULONG NextEntryOffset;
87 ULONG NumberOfThreads;
88 LARGE_INTEGER Reserved[3];
89 LARGE_INTEGER CreateTime;
90 LARGE_INTEGER UserTime;
91 LARGE_INTEGER KernelTime;
92 UNICODE_STRING ImageName;
93 DWORD BasePriority;
94 HANDLE ProcessId;
95 HANDLE ParentProcessId;
96 ULONG HandleCount;
97 ULONG Reserved2[2];
98 VM_COUNTERS VMCounters;
99 IO_COUNTERS IOCounters;
100 SYSTEM_THREAD Threads[1];
101 }; // SYSTEM_PROCESS_INFORMATION
102 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
103
104 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
107 #if KMP_ARCH_X86
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
113 #else
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
119 #endif
120
121 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
122 PVOID, ULONG, PULONG);
123 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
124
125 HMODULE ntdll = NULL;
126
127 /* End of NtQuerySystemInformation()-related code */
128
129 static HMODULE kernel32 = NULL;
130
131 #if KMP_HANDLE_SIGNALS
132 typedef void (*sig_func_t)(int);
133 static sig_func_t __kmp_sighldrs[NSIG];
134 static int __kmp_siginstalled[NSIG];
135 #endif
136
137 #if KMP_USE_MONITOR
138 static HANDLE __kmp_monitor_ev;
139 #endif
140 static kmp_int64 __kmp_win32_time;
141 double __kmp_win32_tick;
142
143 int __kmp_init_runtime = FALSE;
144 CRITICAL_SECTION __kmp_win32_section;
145
__kmp_win32_mutex_init(kmp_win32_mutex_t * mx)146 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
147 InitializeCriticalSection(&mx->cs);
148 #if USE_ITT_BUILD
149 __kmp_itt_system_object_created(&mx->cs, "Critical Section");
150 #endif /* USE_ITT_BUILD */
151 }
152
__kmp_win32_mutex_destroy(kmp_win32_mutex_t * mx)153 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
154 DeleteCriticalSection(&mx->cs);
155 }
156
__kmp_win32_mutex_lock(kmp_win32_mutex_t * mx)157 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
158 EnterCriticalSection(&mx->cs);
159 }
160
__kmp_win32_mutex_trylock(kmp_win32_mutex_t * mx)161 int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
162 return TryEnterCriticalSection(&mx->cs);
163 }
164
__kmp_win32_mutex_unlock(kmp_win32_mutex_t * mx)165 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
166 LeaveCriticalSection(&mx->cs);
167 }
168
__kmp_win32_cond_init(kmp_win32_cond_t * cv)169 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
170 cv->waiters_count_ = 0;
171 cv->wait_generation_count_ = 0;
172 cv->release_count_ = 0;
173
174 /* Initialize the critical section */
175 __kmp_win32_mutex_init(&cv->waiters_count_lock_);
176
177 /* Create a manual-reset event. */
178 cv->event_ = CreateEvent(NULL, // no security
179 TRUE, // manual-reset
180 FALSE, // non-signaled initially
181 NULL); // unnamed
182 #if USE_ITT_BUILD
183 __kmp_itt_system_object_created(cv->event_, "Event");
184 #endif /* USE_ITT_BUILD */
185 }
186
__kmp_win32_cond_destroy(kmp_win32_cond_t * cv)187 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
188 __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
189 __kmp_free_handle(cv->event_);
190 memset(cv, '\0', sizeof(*cv));
191 }
192
193 /* TODO associate cv with a team instead of a thread so as to optimize
194 the case where we wake up a whole team */
195
196 template <class C>
__kmp_win32_cond_wait(kmp_win32_cond_t * cv,kmp_win32_mutex_t * mx,kmp_info_t * th,C * flag)197 static void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
198 kmp_info_t *th, C *flag) {
199 int my_generation;
200 int last_waiter;
201
202 /* Avoid race conditions */
203 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
204
205 /* Increment count of waiters */
206 cv->waiters_count_++;
207
208 /* Store current generation in our activation record. */
209 my_generation = cv->wait_generation_count_;
210
211 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
212 __kmp_win32_mutex_unlock(mx);
213
214 for (;;) {
215 int wait_done = 0;
216 DWORD res, timeout = 5000; // just tried to quess an appropriate number
217 /* Wait until the event is signaled */
218 res = WaitForSingleObject(cv->event_, timeout);
219
220 if (res == WAIT_OBJECT_0) {
221 // event signaled
222 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
223 /* Exit the loop when the <cv->event_> is signaled and there are still
224 waiting threads from this <wait_generation> that haven't been released
225 from this wait yet. */
226 wait_done = (cv->release_count_ > 0) &&
227 (cv->wait_generation_count_ != my_generation);
228 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
229 } else if (res == WAIT_TIMEOUT || res == WAIT_FAILED) {
230 // check if the flag and cv counters are in consistent state
231 // as MS sent us debug dump whith inconsistent state of data
232 __kmp_win32_mutex_lock(mx);
233 typename C::flag_t old_f = flag->set_sleeping();
234 if (!flag->done_check_val(old_f & ~KMP_BARRIER_SLEEP_STATE)) {
235 __kmp_win32_mutex_unlock(mx);
236 continue;
237 }
238 // condition fulfilled, exiting
239 old_f = flag->unset_sleeping();
240 KMP_DEBUG_ASSERT(old_f & KMP_BARRIER_SLEEP_STATE);
241 TCW_PTR(th->th.th_sleep_loc, NULL);
242 KF_TRACE(50, ("__kmp_win32_cond_wait: exiting, condition "
243 "fulfilled: flag's loc(%p): %u => %u\n",
244 flag->get(), old_f, *(flag->get())));
245
246 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
247 KMP_DEBUG_ASSERT(cv->waiters_count_ > 0);
248 cv->release_count_ = cv->waiters_count_;
249 cv->wait_generation_count_++;
250 wait_done = 1;
251 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
252
253 __kmp_win32_mutex_unlock(mx);
254 }
255 /* there used to be a semicolon after the if statement, it looked like a
256 bug, so i removed it */
257 if (wait_done)
258 break;
259 }
260
261 __kmp_win32_mutex_lock(mx);
262 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
263
264 cv->waiters_count_--;
265 cv->release_count_--;
266
267 last_waiter = (cv->release_count_ == 0);
268
269 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
270
271 if (last_waiter) {
272 /* We're the last waiter to be notified, so reset the manual event. */
273 ResetEvent(cv->event_);
274 }
275 }
276
__kmp_win32_cond_broadcast(kmp_win32_cond_t * cv)277 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
278 __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
279
280 if (cv->waiters_count_ > 0) {
281 SetEvent(cv->event_);
282 /* Release all the threads in this generation. */
283
284 cv->release_count_ = cv->waiters_count_;
285
286 /* Start a new generation. */
287 cv->wait_generation_count_++;
288 }
289
290 __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
291 }
292
__kmp_win32_cond_signal(kmp_win32_cond_t * cv)293 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
294 __kmp_win32_cond_broadcast(cv);
295 }
296
__kmp_enable(int new_state)297 void __kmp_enable(int new_state) {
298 if (__kmp_init_runtime)
299 LeaveCriticalSection(&__kmp_win32_section);
300 }
301
__kmp_disable(int * old_state)302 void __kmp_disable(int *old_state) {
303 *old_state = 0;
304
305 if (__kmp_init_runtime)
306 EnterCriticalSection(&__kmp_win32_section);
307 }
308
__kmp_suspend_initialize(void)309 void __kmp_suspend_initialize(void) { /* do nothing */
310 }
311
__kmp_suspend_initialize_thread(kmp_info_t * th)312 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
313 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init);
314 int new_value = TRUE;
315 // Return if already initialized
316 if (old_value == new_value)
317 return;
318 // Wait, then return if being initialized
319 if (old_value == -1 ||
320 !__kmp_atomic_compare_store(&th->th.th_suspend_init, old_value, -1)) {
321 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init) != new_value) {
322 KMP_CPU_PAUSE();
323 }
324 } else {
325 // Claim to be the initializer and do initializations
326 __kmp_win32_cond_init(&th->th.th_suspend_cv);
327 __kmp_win32_mutex_init(&th->th.th_suspend_mx);
328 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, new_value);
329 }
330 }
331
__kmp_suspend_uninitialize_thread(kmp_info_t * th)332 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
333 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init)) {
334 /* this means we have initialize the suspension pthread objects for this
335 thread in this instance of the process */
336 __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
337 __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
338 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init, FALSE);
339 }
340 }
341
__kmp_try_suspend_mx(kmp_info_t * th)342 int __kmp_try_suspend_mx(kmp_info_t *th) {
343 return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
344 }
345
__kmp_lock_suspend_mx(kmp_info_t * th)346 void __kmp_lock_suspend_mx(kmp_info_t *th) {
347 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
348 }
349
__kmp_unlock_suspend_mx(kmp_info_t * th)350 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
351 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
352 }
353
354 /* This routine puts the calling thread to sleep after setting the
355 sleep bit for the indicated flag variable to true. */
356 template <class C>
__kmp_suspend_template(int th_gtid,C * flag)357 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
358 kmp_info_t *th = __kmp_threads[th_gtid];
359 int status;
360 typename C::flag_t old_spin;
361
362 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
363 th_gtid, flag->get()));
364
365 __kmp_suspend_initialize_thread(th);
366 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
367
368 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
369 " loc(%p)\n",
370 th_gtid, flag->get()));
371
372 /* TODO: shouldn't this use release semantics to ensure that
373 __kmp_suspend_initialize_thread gets called first? */
374 old_spin = flag->set_sleeping();
375 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
376 __kmp_pause_status != kmp_soft_paused) {
377 flag->unset_sleeping();
378 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
379 return;
380 }
381
382 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
383 " loc(%p)==%d\n",
384 th_gtid, flag->get(), *(flag->get())));
385
386 if (flag->done_check_val(old_spin)) {
387 old_spin = flag->unset_sleeping();
388 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
389 "for flag's loc(%p)\n",
390 th_gtid, flag->get()));
391 } else {
392 #ifdef DEBUG_SUSPEND
393 __kmp_suspend_count++;
394 #endif
395 /* Encapsulate in a loop as the documentation states that this may "with
396 low probability" return when the condition variable has not been signaled
397 or broadcast */
398 int deactivated = FALSE;
399 TCW_PTR(th->th.th_sleep_loc, (void *)flag);
400 while (flag->is_sleeping()) {
401 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
402 "kmp_win32_cond_wait()\n",
403 th_gtid));
404 // Mark the thread as no longer active (only in the first iteration of the
405 // loop).
406 if (!deactivated) {
407 th->th.th_active = FALSE;
408 if (th->th.th_active_in_pool) {
409 th->th.th_active_in_pool = FALSE;
410 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
411 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
412 }
413 deactivated = TRUE;
414 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
415 flag);
416 } else {
417 __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, th,
418 flag);
419 }
420
421 #ifdef KMP_DEBUG
422 if (flag->is_sleeping()) {
423 KF_TRACE(100,
424 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
425 }
426 #endif /* KMP_DEBUG */
427
428 } // while
429
430 // Mark the thread as active again (if it was previous marked as inactive)
431 if (deactivated) {
432 th->th.th_active = TRUE;
433 if (TCR_4(th->th.th_in_pool)) {
434 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
435 th->th.th_active_in_pool = TRUE;
436 }
437 }
438 }
439
440 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
441
442 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
443 }
444
__kmp_suspend_32(int th_gtid,kmp_flag_32 * flag)445 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
446 __kmp_suspend_template(th_gtid, flag);
447 }
__kmp_suspend_64(int th_gtid,kmp_flag_64 * flag)448 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
449 __kmp_suspend_template(th_gtid, flag);
450 }
__kmp_suspend_oncore(int th_gtid,kmp_flag_oncore * flag)451 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
452 __kmp_suspend_template(th_gtid, flag);
453 }
454
455 /* This routine signals the thread specified by target_gtid to wake up
456 after setting the sleep bit indicated by the flag argument to FALSE */
457 template <class C>
__kmp_resume_template(int target_gtid,C * flag)458 static inline void __kmp_resume_template(int target_gtid, C *flag) {
459 kmp_info_t *th = __kmp_threads[target_gtid];
460 int status;
461
462 #ifdef KMP_DEBUG
463 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
464 #endif
465
466 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
467 gtid, target_gtid));
468
469 __kmp_suspend_initialize_thread(th);
470 __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
471
472 if (!flag) { // coming from __kmp_null_resume_wrapper
473 flag = (C *)th->th.th_sleep_loc;
474 }
475
476 // First, check if the flag is null or its type has changed. If so, someone
477 // else woke it up.
478 if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
479 // simply shows what
480 // flag was cast to
481 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
482 "awake: flag's loc(%p)\n",
483 gtid, target_gtid, NULL));
484 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
485 return;
486 } else {
487 typename C::flag_t old_spin = flag->unset_sleeping();
488 if (!flag->is_sleeping_val(old_spin)) {
489 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
490 "awake: flag's loc(%p): %u => %u\n",
491 gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
492 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
493 return;
494 }
495 }
496 TCW_PTR(th->th.th_sleep_loc, NULL);
497 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
498 "bit for flag's loc(%p)\n",
499 gtid, target_gtid, flag->get()));
500
501 __kmp_win32_cond_signal(&th->th.th_suspend_cv);
502 __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
503
504 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
505 " for T#%d\n",
506 gtid, target_gtid));
507 }
508
__kmp_resume_32(int target_gtid,kmp_flag_32 * flag)509 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
510 __kmp_resume_template(target_gtid, flag);
511 }
__kmp_resume_64(int target_gtid,kmp_flag_64 * flag)512 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
513 __kmp_resume_template(target_gtid, flag);
514 }
__kmp_resume_oncore(int target_gtid,kmp_flag_oncore * flag)515 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
516 __kmp_resume_template(target_gtid, flag);
517 }
518
__kmp_yield()519 void __kmp_yield() { Sleep(0); }
520
__kmp_gtid_set_specific(int gtid)521 void __kmp_gtid_set_specific(int gtid) {
522 if (__kmp_init_gtid) {
523 KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
524 __kmp_gtid_threadprivate_key));
525 if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
526 KMP_FATAL(TLSSetValueFailed);
527 } else {
528 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
529 }
530 }
531
__kmp_gtid_get_specific()532 int __kmp_gtid_get_specific() {
533 int gtid;
534 if (!__kmp_init_gtid) {
535 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
536 "KMP_GTID_SHUTDOWN\n"));
537 return KMP_GTID_SHUTDOWN;
538 }
539 gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
540 if (gtid == 0) {
541 gtid = KMP_GTID_DNE;
542 } else {
543 gtid--;
544 }
545 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
546 __kmp_gtid_threadprivate_key, gtid));
547 return gtid;
548 }
549
__kmp_affinity_bind_thread(int proc)550 void __kmp_affinity_bind_thread(int proc) {
551 if (__kmp_num_proc_groups > 1) {
552 // Form the GROUP_AFFINITY struct directly, rather than filling
553 // out a bit vector and calling __kmp_set_system_affinity().
554 GROUP_AFFINITY ga;
555 KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
556 sizeof(DWORD_PTR))));
557 ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
558 ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
559 ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
560
561 KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
562 if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
563 DWORD error = GetLastError();
564 if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
565 kmp_msg_t err_code = KMP_ERR(error);
566 __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
567 __kmp_msg_null);
568 if (__kmp_generate_warnings == kmp_warnings_off) {
569 __kmp_str_free(&err_code.str);
570 }
571 }
572 }
573 } else {
574 kmp_affin_mask_t *mask;
575 KMP_CPU_ALLOC_ON_STACK(mask);
576 KMP_CPU_ZERO(mask);
577 KMP_CPU_SET(proc, mask);
578 __kmp_set_system_affinity(mask, TRUE);
579 KMP_CPU_FREE_FROM_STACK(mask);
580 }
581 }
582
__kmp_affinity_determine_capable(const char * env_var)583 void __kmp_affinity_determine_capable(const char *env_var) {
584 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
585
586 #if KMP_GROUP_AFFINITY
587 KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
588 #else
589 KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
590 #endif
591
592 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
593 "Windows* OS affinity interface functional (mask size = "
594 "%" KMP_SIZE_T_SPEC ").\n",
595 __kmp_affin_mask_size));
596 }
597
__kmp_read_cpu_time(void)598 double __kmp_read_cpu_time(void) {
599 FILETIME CreationTime, ExitTime, KernelTime, UserTime;
600 int status;
601 double cpu_time;
602
603 cpu_time = 0;
604
605 status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
606 &KernelTime, &UserTime);
607
608 if (status) {
609 double sec = 0;
610
611 sec += KernelTime.dwHighDateTime;
612 sec += UserTime.dwHighDateTime;
613
614 /* Shift left by 32 bits */
615 sec *= (double)(1 << 16) * (double)(1 << 16);
616
617 sec += KernelTime.dwLowDateTime;
618 sec += UserTime.dwLowDateTime;
619
620 cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
621 }
622
623 return cpu_time;
624 }
625
__kmp_read_system_info(struct kmp_sys_info * info)626 int __kmp_read_system_info(struct kmp_sys_info *info) {
627 info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
628 info->minflt = 0; /* the number of page faults serviced without any I/O */
629 info->majflt = 0; /* the number of page faults serviced that required I/O */
630 info->nswap = 0; // the number of times a process was "swapped" out of memory
631 info->inblock = 0; // the number of times the file system had to perform input
632 info->oublock = 0; // number of times the file system had to perform output
633 info->nvcsw = 0; /* the number of times a context switch was voluntarily */
634 info->nivcsw = 0; /* the number of times a context switch was forced */
635
636 return 1;
637 }
638
__kmp_runtime_initialize(void)639 void __kmp_runtime_initialize(void) {
640 SYSTEM_INFO info;
641 kmp_str_buf_t path;
642 UINT path_size;
643
644 if (__kmp_init_runtime) {
645 return;
646 }
647
648 #if KMP_DYNAMIC_LIB
649 /* Pin dynamic library for the lifetime of application */
650 {
651 // First, turn off error message boxes
652 UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
653 HMODULE h;
654 BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
655 GET_MODULE_HANDLE_EX_FLAG_PIN,
656 (LPCTSTR)&__kmp_serial_initialize, &h);
657 KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
658 SetErrorMode(err_mode); // Restore error mode
659 KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
660 }
661 #endif
662
663 InitializeCriticalSection(&__kmp_win32_section);
664 #if USE_ITT_BUILD
665 __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
666 #endif /* USE_ITT_BUILD */
667 __kmp_initialize_system_tick();
668
669 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
670 if (!__kmp_cpuinfo.initialized) {
671 __kmp_query_cpuid(&__kmp_cpuinfo);
672 }
673 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
674
675 /* Set up minimum number of threads to switch to TLS gtid */
676 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
677 // Windows* OS, static library.
678 /* New thread may use stack space previously used by another thread,
679 currently terminated. On Windows* OS, in case of static linking, we do not
680 know the moment of thread termination, and our structures (__kmp_threads
681 and __kmp_root arrays) are still keep info about dead threads. This leads
682 to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
683 (by searching through stack addresses of all known threads) for
684 unregistered foreign tread.
685
686 Setting __kmp_tls_gtid_min to 0 workarounds this problem:
687 __kmp_get_global_thread_id() does not search through stacks, but get gtid
688 from TLS immediately.
689 --ln
690 */
691 __kmp_tls_gtid_min = 0;
692 #else
693 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
694 #endif
695
696 /* for the static library */
697 if (!__kmp_gtid_threadprivate_key) {
698 __kmp_gtid_threadprivate_key = TlsAlloc();
699 if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
700 KMP_FATAL(TLSOutOfIndexes);
701 }
702 }
703
704 // Load ntdll.dll.
705 /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
706 (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
707 have to specify full path to the library. */
708 __kmp_str_buf_init(&path);
709 path_size = GetSystemDirectory(path.str, path.size);
710 KMP_DEBUG_ASSERT(path_size > 0);
711 if (path_size >= path.size) {
712 // Buffer is too short. Expand the buffer and try again.
713 __kmp_str_buf_reserve(&path, path_size);
714 path_size = GetSystemDirectory(path.str, path.size);
715 KMP_DEBUG_ASSERT(path_size > 0);
716 }
717 if (path_size > 0 && path_size < path.size) {
718 // Now we have system directory name in the buffer.
719 // Append backslash and name of dll to form full path,
720 path.used = path_size;
721 __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
722
723 // Now load ntdll using full path.
724 ntdll = GetModuleHandle(path.str);
725 }
726
727 KMP_DEBUG_ASSERT(ntdll != NULL);
728 if (ntdll != NULL) {
729 NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
730 ntdll, "NtQuerySystemInformation");
731 }
732 KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
733
734 #if KMP_GROUP_AFFINITY
735 // Load kernel32.dll.
736 // Same caveat - must use full system path name.
737 if (path_size > 0 && path_size < path.size) {
738 // Truncate the buffer back to just the system path length,
739 // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
740 path.used = path_size;
741 __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
742
743 // Load kernel32.dll using full path.
744 kernel32 = GetModuleHandle(path.str);
745 KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
746
747 // Load the function pointers to kernel32.dll routines
748 // that may or may not exist on this system.
749 if (kernel32 != NULL) {
750 __kmp_GetActiveProcessorCount =
751 (kmp_GetActiveProcessorCount_t)GetProcAddress(
752 kernel32, "GetActiveProcessorCount");
753 __kmp_GetActiveProcessorGroupCount =
754 (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
755 kernel32, "GetActiveProcessorGroupCount");
756 __kmp_GetThreadGroupAffinity =
757 (kmp_GetThreadGroupAffinity_t)GetProcAddress(
758 kernel32, "GetThreadGroupAffinity");
759 __kmp_SetThreadGroupAffinity =
760 (kmp_SetThreadGroupAffinity_t)GetProcAddress(
761 kernel32, "SetThreadGroupAffinity");
762
763 KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
764 " = %p\n",
765 __kmp_GetActiveProcessorCount));
766 KA_TRACE(10, ("__kmp_runtime_initialize: "
767 "__kmp_GetActiveProcessorGroupCount = %p\n",
768 __kmp_GetActiveProcessorGroupCount));
769 KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
770 " = %p\n",
771 __kmp_GetThreadGroupAffinity));
772 KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
773 " = %p\n",
774 __kmp_SetThreadGroupAffinity));
775 KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
776 sizeof(kmp_affin_mask_t)));
777
778 // See if group affinity is supported on this system.
779 // If so, calculate the #groups and #procs.
780 //
781 // Group affinity was introduced with Windows* 7 OS and
782 // Windows* Server 2008 R2 OS.
783 if ((__kmp_GetActiveProcessorCount != NULL) &&
784 (__kmp_GetActiveProcessorGroupCount != NULL) &&
785 (__kmp_GetThreadGroupAffinity != NULL) &&
786 (__kmp_SetThreadGroupAffinity != NULL) &&
787 ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
788 1)) {
789 // Calculate the total number of active OS procs.
790 int i;
791
792 KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
793 " detected\n",
794 __kmp_num_proc_groups));
795
796 __kmp_xproc = 0;
797
798 for (i = 0; i < __kmp_num_proc_groups; i++) {
799 DWORD size = __kmp_GetActiveProcessorCount(i);
800 __kmp_xproc += size;
801 KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
802 i, size));
803 }
804 } else {
805 KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
806 " detected\n",
807 __kmp_num_proc_groups));
808 }
809 }
810 }
811 if (__kmp_num_proc_groups <= 1) {
812 GetSystemInfo(&info);
813 __kmp_xproc = info.dwNumberOfProcessors;
814 }
815 #else
816 GetSystemInfo(&info);
817 __kmp_xproc = info.dwNumberOfProcessors;
818 #endif /* KMP_GROUP_AFFINITY */
819
820 // If the OS said there were 0 procs, take a guess and use a value of 2.
821 // This is done for Linux* OS, also. Do we need error / warning?
822 if (__kmp_xproc <= 0) {
823 __kmp_xproc = 2;
824 }
825
826 KA_TRACE(5,
827 ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
828
829 __kmp_str_buf_free(&path);
830
831 #if USE_ITT_BUILD
832 __kmp_itt_initialize();
833 #endif /* USE_ITT_BUILD */
834
835 __kmp_init_runtime = TRUE;
836 } // __kmp_runtime_initialize
837
__kmp_runtime_destroy(void)838 void __kmp_runtime_destroy(void) {
839 if (!__kmp_init_runtime) {
840 return;
841 }
842
843 #if USE_ITT_BUILD
844 __kmp_itt_destroy();
845 #endif /* USE_ITT_BUILD */
846
847 /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
848 /* due to the KX_TRACE() commands */
849 KA_TRACE(40, ("__kmp_runtime_destroy\n"));
850
851 if (__kmp_gtid_threadprivate_key) {
852 TlsFree(__kmp_gtid_threadprivate_key);
853 __kmp_gtid_threadprivate_key = 0;
854 }
855
856 __kmp_affinity_uninitialize();
857 DeleteCriticalSection(&__kmp_win32_section);
858
859 ntdll = NULL;
860 NtQuerySystemInformation = NULL;
861
862 #if KMP_ARCH_X86_64
863 kernel32 = NULL;
864 __kmp_GetActiveProcessorCount = NULL;
865 __kmp_GetActiveProcessorGroupCount = NULL;
866 __kmp_GetThreadGroupAffinity = NULL;
867 __kmp_SetThreadGroupAffinity = NULL;
868 #endif // KMP_ARCH_X86_64
869
870 __kmp_init_runtime = FALSE;
871 }
872
__kmp_terminate_thread(int gtid)873 void __kmp_terminate_thread(int gtid) {
874 kmp_info_t *th = __kmp_threads[gtid];
875
876 if (!th)
877 return;
878
879 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
880
881 if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
882 /* It's OK, the thread may have exited already */
883 }
884 __kmp_free_handle(th->th.th_info.ds.ds_thread);
885 }
886
__kmp_clear_system_time(void)887 void __kmp_clear_system_time(void) {
888 BOOL status;
889 LARGE_INTEGER time;
890 status = QueryPerformanceCounter(&time);
891 __kmp_win32_time = (kmp_int64)time.QuadPart;
892 }
893
__kmp_initialize_system_tick(void)894 void __kmp_initialize_system_tick(void) {
895 {
896 BOOL status;
897 LARGE_INTEGER freq;
898
899 status = QueryPerformanceFrequency(&freq);
900 if (!status) {
901 DWORD error = GetLastError();
902 __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
903 KMP_ERR(error), __kmp_msg_null);
904
905 } else {
906 __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
907 }
908 }
909 }
910
911 /* Calculate the elapsed wall clock time for the user */
912
__kmp_elapsed(double * t)913 void __kmp_elapsed(double *t) {
914 BOOL status;
915 LARGE_INTEGER now;
916 status = QueryPerformanceCounter(&now);
917 *t = ((double)now.QuadPart) * __kmp_win32_tick;
918 }
919
920 /* Calculate the elapsed wall clock tick for the user */
921
__kmp_elapsed_tick(double * t)922 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
923
__kmp_read_system_time(double * delta)924 void __kmp_read_system_time(double *delta) {
925 if (delta != NULL) {
926 BOOL status;
927 LARGE_INTEGER now;
928
929 status = QueryPerformanceCounter(&now);
930
931 *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
932 __kmp_win32_tick;
933 }
934 }
935
936 /* Return the current time stamp in nsec */
__kmp_now_nsec()937 kmp_uint64 __kmp_now_nsec() {
938 LARGE_INTEGER now;
939 QueryPerformanceCounter(&now);
940 return 1e9 * __kmp_win32_tick * now.QuadPart;
941 }
942
943 extern "C"
__kmp_launch_worker(void * arg)944 void *__stdcall __kmp_launch_worker(void *arg) {
945 volatile void *stack_data;
946 void *exit_val;
947 void *padding = 0;
948 kmp_info_t *this_thr = (kmp_info_t *)arg;
949 int gtid;
950
951 gtid = this_thr->th.th_info.ds.ds_gtid;
952 __kmp_gtid_set_specific(gtid);
953 #ifdef KMP_TDATA_GTID
954 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
955 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
956 "reference: http://support.microsoft.com/kb/118816"
957 //__kmp_gtid = gtid;
958 #endif
959
960 #if USE_ITT_BUILD
961 __kmp_itt_thread_name(gtid);
962 #endif /* USE_ITT_BUILD */
963
964 __kmp_affinity_set_init_mask(gtid, FALSE);
965
966 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
967 // Set FP control regs to be a copy of the parallel initialization thread's.
968 __kmp_clear_x87_fpu_status_word();
969 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
970 __kmp_load_mxcsr(&__kmp_init_mxcsr);
971 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
972
973 if (__kmp_stkoffset > 0 && gtid > 0) {
974 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
975 }
976
977 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
978 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
979 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
980
981 if (TCR_4(__kmp_gtid_mode) <
982 2) { // check stack only if it is used to get gtid
983 TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
984 KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
985 __kmp_check_stack_overlap(this_thr);
986 }
987 KMP_MB();
988 exit_val = __kmp_launch_thread(this_thr);
989 KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
990 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
991 KMP_MB();
992 return exit_val;
993 }
994
995 #if KMP_USE_MONITOR
996 /* The monitor thread controls all of the threads in the complex */
997
__kmp_launch_monitor(void * arg)998 void *__stdcall __kmp_launch_monitor(void *arg) {
999 DWORD wait_status;
1000 kmp_thread_t monitor;
1001 int status;
1002 int interval;
1003 kmp_info_t *this_thr = (kmp_info_t *)arg;
1004
1005 KMP_DEBUG_ASSERT(__kmp_init_monitor);
1006 TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
1007 // TODO: hide "2" in enum (like {true,false,started})
1008 this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1009 TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
1010
1011 KMP_MB(); /* Flush all pending memory write invalidates. */
1012 KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
1013
1014 monitor = GetCurrentThread();
1015
1016 /* set thread priority */
1017 status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
1018 if (!status) {
1019 DWORD error = GetLastError();
1020 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1021 }
1022
1023 /* register us as monitor */
1024 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
1025 #ifdef KMP_TDATA_GTID
1026 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
1027 "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
1028 "reference: http://support.microsoft.com/kb/118816"
1029 //__kmp_gtid = KMP_GTID_MONITOR;
1030 #endif
1031
1032 #if USE_ITT_BUILD
1033 __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
1034 // monitor thread.
1035 #endif /* USE_ITT_BUILD */
1036
1037 KMP_MB(); /* Flush all pending memory write invalidates. */
1038
1039 interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
1040
1041 while (!TCR_4(__kmp_global.g.g_done)) {
1042 /* This thread monitors the state of the system */
1043
1044 KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
1045
1046 wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
1047
1048 if (wait_status == WAIT_TIMEOUT) {
1049 TCW_4(__kmp_global.g.g_time.dt.t_value,
1050 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1051 }
1052
1053 KMP_MB(); /* Flush all pending memory write invalidates. */
1054 }
1055
1056 KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1057
1058 status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1059 if (!status) {
1060 DWORD error = GetLastError();
1061 __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1062 }
1063
1064 if (__kmp_global.g.g_abort != 0) {
1065 /* now we need to terminate the worker threads */
1066 /* the value of t_abort is the signal we caught */
1067 int gtid;
1068
1069 KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1070 (__kmp_global.g.g_abort)));
1071
1072 /* terminate the OpenMP worker threads */
1073 /* TODO this is not valid for sibling threads!!
1074 * the uber master might not be 0 anymore.. */
1075 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1076 __kmp_terminate_thread(gtid);
1077
1078 __kmp_cleanup();
1079
1080 Sleep(0);
1081
1082 KA_TRACE(10,
1083 ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1084
1085 if (__kmp_global.g.g_abort > 0) {
1086 raise(__kmp_global.g.g_abort);
1087 }
1088 }
1089
1090 TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1091
1092 KMP_MB();
1093 return arg;
1094 }
1095 #endif
1096
__kmp_create_worker(int gtid,kmp_info_t * th,size_t stack_size)1097 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1098 kmp_thread_t handle;
1099 DWORD idThread;
1100
1101 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1102
1103 th->th.th_info.ds.ds_gtid = gtid;
1104
1105 if (KMP_UBER_GTID(gtid)) {
1106 int stack_data;
1107
1108 /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1109 other threads to use. Is it appropriate to just use GetCurrentThread?
1110 When should we close this handle? When unregistering the root? */
1111 {
1112 BOOL rc;
1113 rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1114 GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1115 FALSE, DUPLICATE_SAME_ACCESS);
1116 KMP_ASSERT(rc);
1117 KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1118 "handle = %" KMP_UINTPTR_SPEC "\n",
1119 (LPVOID)th, th->th.th_info.ds.ds_thread));
1120 th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1121 }
1122 if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1123 /* we will dynamically update the stack range if gtid_mode == 1 */
1124 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1125 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1126 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1127 __kmp_check_stack_overlap(th);
1128 }
1129 } else {
1130 KMP_MB(); /* Flush all pending memory write invalidates. */
1131
1132 /* Set stack size for this thread now. */
1133 KA_TRACE(10,
1134 ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1135 stack_size));
1136
1137 stack_size += gtid * __kmp_stkoffset;
1138
1139 TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1140 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1141
1142 KA_TRACE(10,
1143 ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1144 " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1145 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1146 (LPVOID)th, &idThread));
1147
1148 handle = CreateThread(
1149 NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1150 (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1151
1152 KA_TRACE(10,
1153 ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1154 " bytes, &__kmp_launch_worker = %p, th = %p, "
1155 "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1156 (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1157 (LPVOID)th, idThread, handle));
1158
1159 if (handle == 0) {
1160 DWORD error = GetLastError();
1161 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1162 } else {
1163 th->th.th_info.ds.ds_thread = handle;
1164 }
1165
1166 KMP_MB(); /* Flush all pending memory write invalidates. */
1167 }
1168
1169 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1170 }
1171
__kmp_still_running(kmp_info_t * th)1172 int __kmp_still_running(kmp_info_t *th) {
1173 return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1174 }
1175
1176 #if KMP_USE_MONITOR
__kmp_create_monitor(kmp_info_t * th)1177 void __kmp_create_monitor(kmp_info_t *th) {
1178 kmp_thread_t handle;
1179 DWORD idThread;
1180 int ideal, new_ideal;
1181
1182 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1183 // We don't need monitor thread in case of MAX_BLOCKTIME
1184 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1185 "MAX blocktime\n"));
1186 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1187 th->th.th_info.ds.ds_gtid = 0;
1188 TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1189 return;
1190 }
1191 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1192
1193 KMP_MB(); /* Flush all pending memory write invalidates. */
1194
1195 __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1196 if (__kmp_monitor_ev == NULL) {
1197 DWORD error = GetLastError();
1198 __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1199 }
1200 #if USE_ITT_BUILD
1201 __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1202 #endif /* USE_ITT_BUILD */
1203
1204 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1205 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1206
1207 // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1208 // to automatically expand stacksize based on CreateThread error code.
1209 if (__kmp_monitor_stksize == 0) {
1210 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1211 }
1212 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1213 __kmp_monitor_stksize = __kmp_sys_min_stksize;
1214 }
1215
1216 KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1217 (int)__kmp_monitor_stksize));
1218
1219 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1220
1221 handle =
1222 CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1223 (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1224 STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1225 if (handle == 0) {
1226 DWORD error = GetLastError();
1227 __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1228 } else
1229 th->th.th_info.ds.ds_thread = handle;
1230
1231 KMP_MB(); /* Flush all pending memory write invalidates. */
1232
1233 KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1234 (void *)th->th.th_info.ds.ds_thread));
1235 }
1236 #endif
1237
1238 /* Check to see if thread is still alive.
1239 NOTE: The ExitProcess(code) system call causes all threads to Terminate
1240 with a exit_val = code. Because of this we can not rely on exit_val having
1241 any particular value. So this routine may return STILL_ALIVE in exit_val
1242 even after the thread is dead. */
1243
__kmp_is_thread_alive(kmp_info_t * th,DWORD * exit_val)1244 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1245 DWORD rc;
1246 rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1247 if (rc == 0) {
1248 DWORD error = GetLastError();
1249 __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
1250 __kmp_msg_null);
1251 }
1252 return (*exit_val == STILL_ACTIVE);
1253 }
1254
__kmp_exit_thread(int exit_status)1255 void __kmp_exit_thread(int exit_status) {
1256 ExitThread(exit_status);
1257 } // __kmp_exit_thread
1258
1259 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
__kmp_reap_common(kmp_info_t * th)1260 static void __kmp_reap_common(kmp_info_t *th) {
1261 DWORD exit_val;
1262
1263 KMP_MB(); /* Flush all pending memory write invalidates. */
1264
1265 KA_TRACE(
1266 10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1267
1268 /* 2006-10-19:
1269 There are two opposite situations:
1270 1. Windows* OS keep thread alive after it resets ds_alive flag and
1271 exits from thread function. (For example, see C70770/Q394281 "unloading of
1272 dll based on OMP is very slow".)
1273 2. Windows* OS may kill thread before it resets ds_alive flag.
1274
1275 Right solution seems to be waiting for *either* thread termination *or*
1276 ds_alive resetting. */
1277 {
1278 // TODO: This code is very similar to KMP_WAIT. Need to generalize
1279 // KMP_WAIT to cover this usage also.
1280 void *obj = NULL;
1281 kmp_uint32 spins;
1282 #if USE_ITT_BUILD
1283 KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1284 #endif /* USE_ITT_BUILD */
1285 KMP_INIT_YIELD(spins);
1286 do {
1287 #if USE_ITT_BUILD
1288 KMP_FSYNC_SPIN_PREPARE(obj);
1289 #endif /* USE_ITT_BUILD */
1290 __kmp_is_thread_alive(th, &exit_val);
1291 KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
1292 } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1293 #if USE_ITT_BUILD
1294 if (exit_val == STILL_ACTIVE) {
1295 KMP_FSYNC_CANCEL(obj);
1296 } else {
1297 KMP_FSYNC_SPIN_ACQUIRED(obj);
1298 }
1299 #endif /* USE_ITT_BUILD */
1300 }
1301
1302 __kmp_free_handle(th->th.th_info.ds.ds_thread);
1303
1304 /* NOTE: The ExitProcess(code) system call causes all threads to Terminate
1305 with a exit_val = code. Because of this we can not rely on exit_val having
1306 any particular value. */
1307 if (exit_val == STILL_ACTIVE) {
1308 KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1309 } else if ((void *)exit_val != (void *)th) {
1310 KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1311 }
1312
1313 KA_TRACE(10,
1314 ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1315 "\n",
1316 th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1317
1318 th->th.th_info.ds.ds_thread = 0;
1319 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1320 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1321 th->th.th_info.ds.ds_thread_id = 0;
1322
1323 KMP_MB(); /* Flush all pending memory write invalidates. */
1324 }
1325
1326 #if KMP_USE_MONITOR
__kmp_reap_monitor(kmp_info_t * th)1327 void __kmp_reap_monitor(kmp_info_t *th) {
1328 int status;
1329
1330 KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1331 (void *)th->th.th_info.ds.ds_thread));
1332
1333 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1334 // If both tid and gtid are 0, it means the monitor did not ever start.
1335 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1336 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1337 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1338 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1339 return;
1340 }
1341
1342 KMP_MB(); /* Flush all pending memory write invalidates. */
1343
1344 status = SetEvent(__kmp_monitor_ev);
1345 if (status == FALSE) {
1346 DWORD error = GetLastError();
1347 __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1348 }
1349 KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1350 th->th.th_info.ds.ds_gtid));
1351 __kmp_reap_common(th);
1352
1353 __kmp_free_handle(__kmp_monitor_ev);
1354
1355 KMP_MB(); /* Flush all pending memory write invalidates. */
1356 }
1357 #endif
1358
__kmp_reap_worker(kmp_info_t * th)1359 void __kmp_reap_worker(kmp_info_t *th) {
1360 KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1361 th->th.th_info.ds.ds_gtid));
1362 __kmp_reap_common(th);
1363 }
1364
1365 #if KMP_HANDLE_SIGNALS
1366
__kmp_team_handler(int signo)1367 static void __kmp_team_handler(int signo) {
1368 if (__kmp_global.g.g_abort == 0) {
1369 // Stage 1 signal handler, let's shut down all of the threads.
1370 if (__kmp_debug_buf) {
1371 __kmp_dump_debug_buffer();
1372 }
1373 KMP_MB(); // Flush all pending memory write invalidates.
1374 TCW_4(__kmp_global.g.g_abort, signo);
1375 KMP_MB(); // Flush all pending memory write invalidates.
1376 TCW_4(__kmp_global.g.g_done, TRUE);
1377 KMP_MB(); // Flush all pending memory write invalidates.
1378 }
1379 } // __kmp_team_handler
1380
__kmp_signal(int signum,sig_func_t handler)1381 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1382 sig_func_t old = signal(signum, handler);
1383 if (old == SIG_ERR) {
1384 int error = errno;
1385 __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1386 __kmp_msg_null);
1387 }
1388 return old;
1389 }
1390
__kmp_install_one_handler(int sig,sig_func_t handler,int parallel_init)1391 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1392 int parallel_init) {
1393 sig_func_t old;
1394 KMP_MB(); /* Flush all pending memory write invalidates. */
1395 KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1396 if (parallel_init) {
1397 old = __kmp_signal(sig, handler);
1398 // SIG_DFL on Windows* OS in NULL or 0.
1399 if (old == __kmp_sighldrs[sig]) {
1400 __kmp_siginstalled[sig] = 1;
1401 } else { // Restore/keep user's handler if one previously installed.
1402 old = __kmp_signal(sig, old);
1403 }
1404 } else {
1405 // Save initial/system signal handlers to see if user handlers installed.
1406 // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1407 // called once with parallel_init == TRUE.
1408 old = __kmp_signal(sig, SIG_DFL);
1409 __kmp_sighldrs[sig] = old;
1410 __kmp_signal(sig, old);
1411 }
1412 KMP_MB(); /* Flush all pending memory write invalidates. */
1413 } // __kmp_install_one_handler
1414
__kmp_remove_one_handler(int sig)1415 static void __kmp_remove_one_handler(int sig) {
1416 if (__kmp_siginstalled[sig]) {
1417 sig_func_t old;
1418 KMP_MB(); // Flush all pending memory write invalidates.
1419 KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1420 old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1421 if (old != __kmp_team_handler) {
1422 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1423 "restoring: sig=%d\n",
1424 sig));
1425 old = __kmp_signal(sig, old);
1426 }
1427 __kmp_sighldrs[sig] = NULL;
1428 __kmp_siginstalled[sig] = 0;
1429 KMP_MB(); // Flush all pending memory write invalidates.
1430 }
1431 } // __kmp_remove_one_handler
1432
__kmp_install_signals(int parallel_init)1433 void __kmp_install_signals(int parallel_init) {
1434 KB_TRACE(10, ("__kmp_install_signals: called\n"));
1435 if (!__kmp_handle_signals) {
1436 KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1437 "handlers not installed\n"));
1438 return;
1439 }
1440 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1441 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1442 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1443 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1444 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1445 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1446 } // __kmp_install_signals
1447
__kmp_remove_signals(void)1448 void __kmp_remove_signals(void) {
1449 int sig;
1450 KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1451 for (sig = 1; sig < NSIG; ++sig) {
1452 __kmp_remove_one_handler(sig);
1453 }
1454 } // __kmp_remove_signals
1455
1456 #endif // KMP_HANDLE_SIGNALS
1457
1458 /* Put the thread to sleep for a time period */
__kmp_thread_sleep(int millis)1459 void __kmp_thread_sleep(int millis) {
1460 DWORD status;
1461
1462 status = SleepEx((DWORD)millis, FALSE);
1463 if (status) {
1464 DWORD error = GetLastError();
1465 __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1466 __kmp_msg_null);
1467 }
1468 }
1469
1470 // Determine whether the given address is mapped into the current address space.
__kmp_is_address_mapped(void * addr)1471 int __kmp_is_address_mapped(void *addr) {
1472 DWORD status;
1473 MEMORY_BASIC_INFORMATION lpBuffer;
1474 SIZE_T dwLength;
1475
1476 dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1477
1478 status = VirtualQuery(addr, &lpBuffer, dwLength);
1479
1480 return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1481 ((lpBuffer.Protect == PAGE_NOACCESS) ||
1482 (lpBuffer.Protect == PAGE_EXECUTE)));
1483 }
1484
__kmp_hardware_timestamp(void)1485 kmp_uint64 __kmp_hardware_timestamp(void) {
1486 kmp_uint64 r = 0;
1487
1488 QueryPerformanceCounter((LARGE_INTEGER *)&r);
1489 return r;
1490 }
1491
1492 /* Free handle and check the error code */
__kmp_free_handle(kmp_thread_t tHandle)1493 void __kmp_free_handle(kmp_thread_t tHandle) {
1494 /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1495 * as HANDLE */
1496 BOOL rc;
1497 rc = CloseHandle(tHandle);
1498 if (!rc) {
1499 DWORD error = GetLastError();
1500 __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1501 }
1502 }
1503
__kmp_get_load_balance(int max)1504 int __kmp_get_load_balance(int max) {
1505 static ULONG glb_buff_size = 100 * 1024;
1506
1507 // Saved count of the running threads for the thread balance algorithm
1508 static int glb_running_threads = 0;
1509 static double glb_call_time = 0; /* Thread balance algorithm call time */
1510
1511 int running_threads = 0; // Number of running threads in the system.
1512 NTSTATUS status = 0;
1513 ULONG buff_size = 0;
1514 ULONG info_size = 0;
1515 void *buffer = NULL;
1516 PSYSTEM_PROCESS_INFORMATION spi = NULL;
1517 int first_time = 1;
1518
1519 double call_time = 0.0; // start, finish;
1520
1521 __kmp_elapsed(&call_time);
1522
1523 if (glb_call_time &&
1524 (call_time - glb_call_time < __kmp_load_balance_interval)) {
1525 running_threads = glb_running_threads;
1526 goto finish;
1527 }
1528 glb_call_time = call_time;
1529
1530 // Do not spend time on running algorithm if we have a permanent error.
1531 if (NtQuerySystemInformation == NULL) {
1532 running_threads = -1;
1533 goto finish;
1534 }
1535
1536 if (max <= 0) {
1537 max = INT_MAX;
1538 }
1539
1540 do {
1541
1542 if (first_time) {
1543 buff_size = glb_buff_size;
1544 } else {
1545 buff_size = 2 * buff_size;
1546 }
1547
1548 buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1549 if (buffer == NULL) {
1550 running_threads = -1;
1551 goto finish;
1552 }
1553 status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1554 buff_size, &info_size);
1555 first_time = 0;
1556
1557 } while (status == STATUS_INFO_LENGTH_MISMATCH);
1558 glb_buff_size = buff_size;
1559
1560 #define CHECK(cond) \
1561 { \
1562 KMP_DEBUG_ASSERT(cond); \
1563 if (!(cond)) { \
1564 running_threads = -1; \
1565 goto finish; \
1566 } \
1567 }
1568
1569 CHECK(buff_size >= info_size);
1570 spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1571 for (;;) {
1572 ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1573 CHECK(0 <= offset &&
1574 offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1575 HANDLE pid = spi->ProcessId;
1576 ULONG num = spi->NumberOfThreads;
1577 CHECK(num >= 1);
1578 size_t spi_size =
1579 sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1580 CHECK(offset + spi_size <
1581 info_size); // Make sure process info record fits the buffer.
1582 if (spi->NextEntryOffset != 0) {
1583 CHECK(spi_size <=
1584 spi->NextEntryOffset); // And do not overlap with the next record.
1585 }
1586 // pid == 0 corresponds to the System Idle Process. It always has running
1587 // threads on all cores. So, we don't consider the running threads of this
1588 // process.
1589 if (pid != 0) {
1590 for (int i = 0; i < num; ++i) {
1591 THREAD_STATE state = spi->Threads[i].State;
1592 // Count threads that have Ready or Running state.
1593 // !!! TODO: Why comment does not match the code???
1594 if (state == StateRunning) {
1595 ++running_threads;
1596 // Stop counting running threads if the number is already greater than
1597 // the number of available cores
1598 if (running_threads >= max) {
1599 goto finish;
1600 }
1601 }
1602 }
1603 }
1604 if (spi->NextEntryOffset == 0) {
1605 break;
1606 }
1607 spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1608 }
1609
1610 #undef CHECK
1611
1612 finish: // Clean up and exit.
1613
1614 if (buffer != NULL) {
1615 KMP_INTERNAL_FREE(buffer);
1616 }
1617
1618 glb_running_threads = running_threads;
1619
1620 return running_threads;
1621 } //__kmp_get_load_balance()
1622