15 #include "kmp_affinity.h" 20 #include "kmp_stats.h" 22 #include "kmp_wait_release.h" 23 #include "kmp_wrapper_getpid.h" 25 #if !KMP_OS_FREEBSD && !KMP_OS_NETBSD 29 #include <sys/resource.h> 30 #include <sys/syscall.h> 32 #include <sys/times.h> 35 #if KMP_OS_LINUX && !KMP_OS_CNK 36 #include <sys/sysinfo.h> 51 #include <mach/mach.h> 52 #include <sys/sysctl.h> 54 #include <pthread_np.h> 61 #include "tsan_annotations.h" 63 struct kmp_sys_timer {
64 struct timespec start;
68 #define TS2NS(timespec) (((timespec).tv_sec * 1e9) + (timespec).tv_nsec) 70 static struct kmp_sys_timer __kmp_sys_timer_data;
72 #if KMP_HANDLE_SIGNALS 73 typedef void (*sig_func_t)(int);
74 STATIC_EFI2_WORKAROUND
struct sigaction __kmp_sighldrs[NSIG];
75 static sigset_t __kmp_sigset;
78 static int __kmp_init_runtime = FALSE;
80 static int __kmp_fork_count = 0;
82 static pthread_condattr_t __kmp_suspend_cond_attr;
83 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
85 static kmp_cond_align_t __kmp_wait_cv;
86 static kmp_mutex_align_t __kmp_wait_mx;
88 kmp_uint64 __kmp_ticks_per_msec = 1000000;
91 static void __kmp_print_cond(
char *buffer, kmp_cond_align_t *cond) {
92 KMP_SNPRINTF(buffer, 128,
"(cond (lock (%ld, %d)), (descr (%p)))",
93 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
94 cond->c_cond.__c_waiting);
98 #if (KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED) 102 void __kmp_affinity_bind_thread(
int which) {
103 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
104 "Illegal set affinity operation when not capable");
106 kmp_affin_mask_t *mask;
107 KMP_CPU_ALLOC_ON_STACK(mask);
109 KMP_CPU_SET(which, mask);
110 __kmp_set_system_affinity(mask, TRUE);
111 KMP_CPU_FREE_FROM_STACK(mask);
117 void __kmp_affinity_determine_capable(
const char *env_var) {
120 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024) 125 buf = (
unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
130 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_SIZE_LIMIT, buf);
131 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 132 "initial getaffinity call returned %d errno = %d\n",
138 if (__kmp_affinity_verbose ||
139 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none) &&
140 (__kmp_affinity_type != affinity_default) &&
141 (__kmp_affinity_type != affinity_disabled))) {
143 kmp_msg_t err_code = KMP_ERR(error);
144 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
145 err_code, __kmp_msg_null);
146 if (__kmp_generate_warnings == kmp_warnings_off) {
147 __kmp_str_free(&err_code.str);
150 KMP_AFFINITY_DISABLE();
151 KMP_INTERNAL_FREE(buf);
159 sCode = syscall(__NR_sched_setaffinity, 0, gCode, NULL);
160 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 161 "setaffinity for mask size %d returned %d errno = %d\n",
162 gCode, sCode, errno));
164 if (errno == ENOSYS) {
165 if (__kmp_affinity_verbose ||
166 (__kmp_affinity_warnings &&
167 (__kmp_affinity_type != affinity_none) &&
168 (__kmp_affinity_type != affinity_default) &&
169 (__kmp_affinity_type != affinity_disabled))) {
171 kmp_msg_t err_code = KMP_ERR(error);
172 __kmp_msg(kmp_ms_warning, KMP_MSG(SetAffSysCallNotSupported, env_var),
173 err_code, __kmp_msg_null);
174 if (__kmp_generate_warnings == kmp_warnings_off) {
175 __kmp_str_free(&err_code.str);
178 KMP_AFFINITY_DISABLE();
179 KMP_INTERNAL_FREE(buf);
181 if (errno == EFAULT) {
182 KMP_AFFINITY_ENABLE(gCode);
183 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 184 "affinity supported (mask size %d)\n",
185 (
int)__kmp_affin_mask_size));
186 KMP_INTERNAL_FREE(buf);
194 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 195 "searching for proper set size\n"));
197 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
198 gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
199 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 200 "getaffinity for mask size %d returned %d errno = %d\n",
201 size, gCode, errno));
204 if (errno == ENOSYS) {
206 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 207 "inconsistent OS call behavior: errno == ENOSYS for mask " 210 if (__kmp_affinity_verbose ||
211 (__kmp_affinity_warnings &&
212 (__kmp_affinity_type != affinity_none) &&
213 (__kmp_affinity_type != affinity_default) &&
214 (__kmp_affinity_type != affinity_disabled))) {
216 kmp_msg_t err_code = KMP_ERR(error);
217 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
218 err_code, __kmp_msg_null);
219 if (__kmp_generate_warnings == kmp_warnings_off) {
220 __kmp_str_free(&err_code.str);
223 KMP_AFFINITY_DISABLE();
224 KMP_INTERNAL_FREE(buf);
230 sCode = syscall(__NR_sched_setaffinity, 0, gCode, NULL);
231 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 232 "setaffinity for mask size %d returned %d errno = %d\n",
233 gCode, sCode, errno));
235 if (errno == ENOSYS) {
237 KA_TRACE(30, (
"__kmp_affinity_determine_capable: " 238 "inconsistent OS call behavior: errno == ENOSYS for mask " 241 if (__kmp_affinity_verbose ||
242 (__kmp_affinity_warnings &&
243 (__kmp_affinity_type != affinity_none) &&
244 (__kmp_affinity_type != affinity_default) &&
245 (__kmp_affinity_type != affinity_disabled))) {
247 kmp_msg_t err_code = KMP_ERR(error);
248 __kmp_msg(kmp_ms_warning, KMP_MSG(SetAffSysCallNotSupported, env_var),
249 err_code, __kmp_msg_null);
250 if (__kmp_generate_warnings == kmp_warnings_off) {
251 __kmp_str_free(&err_code.str);
254 KMP_AFFINITY_DISABLE();
255 KMP_INTERNAL_FREE(buf);
258 if (errno == EFAULT) {
259 KMP_AFFINITY_ENABLE(gCode);
260 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 261 "affinity supported (mask size %d)\n",
262 (
int)__kmp_affin_mask_size));
263 KMP_INTERNAL_FREE(buf);
270 KMP_INTERNAL_FREE(buf);
275 KMP_AFFINITY_DISABLE();
276 KA_TRACE(10, (
"__kmp_affinity_determine_capable: " 277 "cannot determine mask size - affinity not supported\n"));
278 if (__kmp_affinity_verbose ||
279 (__kmp_affinity_warnings && (__kmp_affinity_type != affinity_none) &&
280 (__kmp_affinity_type != affinity_default) &&
281 (__kmp_affinity_type != affinity_disabled))) {
282 KMP_WARNING(AffCantGetMaskSize, env_var);
286 #endif // KMP_OS_LINUX && KMP_AFFINITY_SUPPORTED 290 int __kmp_futex_determine_capable() {
292 int rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
293 int retval = (rc == 0) || (errno != ENOSYS);
296 (
"__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
297 KA_TRACE(10, (
"__kmp_futex_determine_capable: futex syscall%s supported\n",
298 retval ?
"" :
" not"));
303 #endif // KMP_USE_FUTEX 305 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (!KMP_ASM_INTRINS) 309 kmp_int8 __kmp_test_then_or8(
volatile kmp_int8 *p, kmp_int8 d) {
310 kmp_int8 old_value, new_value;
312 old_value = TCR_1(*p);
313 new_value = old_value | d;
315 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
317 old_value = TCR_1(*p);
318 new_value = old_value | d;
323 kmp_int8 __kmp_test_then_and8(
volatile kmp_int8 *p, kmp_int8 d) {
324 kmp_int8 old_value, new_value;
326 old_value = TCR_1(*p);
327 new_value = old_value & d;
329 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
331 old_value = TCR_1(*p);
332 new_value = old_value & d;
337 kmp_uint32 __kmp_test_then_or32(
volatile kmp_uint32 *p, kmp_uint32 d) {
338 kmp_uint32 old_value, new_value;
340 old_value = TCR_4(*p);
341 new_value = old_value | d;
343 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
345 old_value = TCR_4(*p);
346 new_value = old_value | d;
351 kmp_uint32 __kmp_test_then_and32(
volatile kmp_uint32 *p, kmp_uint32 d) {
352 kmp_uint32 old_value, new_value;
354 old_value = TCR_4(*p);
355 new_value = old_value & d;
357 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
359 old_value = TCR_4(*p);
360 new_value = old_value & d;
366 kmp_int8 __kmp_test_then_add8(
volatile kmp_int8 *p, kmp_int8 d) {
367 kmp_int8 old_value, new_value;
369 old_value = TCR_1(*p);
370 new_value = old_value + d;
372 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
374 old_value = TCR_1(*p);
375 new_value = old_value + d;
380 kmp_int64 __kmp_test_then_add64(
volatile kmp_int64 *p, kmp_int64 d) {
381 kmp_int64 old_value, new_value;
383 old_value = TCR_8(*p);
384 new_value = old_value + d;
386 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
388 old_value = TCR_8(*p);
389 new_value = old_value + d;
395 kmp_uint64 __kmp_test_then_or64(
volatile kmp_uint64 *p, kmp_uint64 d) {
396 kmp_uint64 old_value, new_value;
398 old_value = TCR_8(*p);
399 new_value = old_value | d;
400 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
402 old_value = TCR_8(*p);
403 new_value = old_value | d;
408 kmp_uint64 __kmp_test_then_and64(
volatile kmp_uint64 *p, kmp_uint64 d) {
409 kmp_uint64 old_value, new_value;
411 old_value = TCR_8(*p);
412 new_value = old_value & d;
413 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
415 old_value = TCR_8(*p);
416 new_value = old_value & d;
423 void __kmp_terminate_thread(
int gtid) {
425 kmp_info_t *th = __kmp_threads[gtid];
430 #ifdef KMP_CANCEL_THREADS 431 KA_TRACE(10, (
"__kmp_terminate_thread: kill (%d)\n", gtid));
432 status = pthread_cancel(th->th.th_info.ds.ds_thread);
433 if (status != 0 && status != ESRCH) {
434 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
445 static kmp_int32 __kmp_set_stack_info(
int gtid, kmp_info_t *th) {
447 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 457 if (!KMP_UBER_GTID(gtid)) {
460 status = pthread_attr_init(&attr);
461 KMP_CHECK_SYSFAIL(
"pthread_attr_init", status);
462 #if KMP_OS_FREEBSD || KMP_OS_NETBSD 463 status = pthread_attr_get_np(pthread_self(), &attr);
464 KMP_CHECK_SYSFAIL(
"pthread_attr_get_np", status);
466 status = pthread_getattr_np(pthread_self(), &attr);
467 KMP_CHECK_SYSFAIL(
"pthread_getattr_np", status);
469 status = pthread_attr_getstack(&attr, &addr, &size);
470 KMP_CHECK_SYSFAIL(
"pthread_attr_getstack", status);
472 (
"__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:" 473 " %lu, low addr: %p\n",
475 status = pthread_attr_destroy(&attr);
476 KMP_CHECK_SYSFAIL(
"pthread_attr_destroy", status);
479 if (size != 0 && addr != 0) {
481 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((
char *)addr) + size));
482 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
483 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
488 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
489 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
490 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
494 static void *__kmp_launch_worker(
void *thr) {
495 int status, old_type, old_state;
496 #ifdef KMP_BLOCK_SIGNALS 497 sigset_t new_set, old_set;
500 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 501 void *
volatile padding = 0;
505 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
506 __kmp_gtid_set_specific(gtid);
507 #ifdef KMP_TDATA_GTID 510 #if KMP_STATS_ENABLED 512 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
514 KMP_SET_THREAD_STATE(IDLE);
519 __kmp_itt_thread_name(gtid);
522 #if KMP_AFFINITY_SUPPORTED 523 __kmp_affinity_set_init_mask(gtid, FALSE);
526 #ifdef KMP_CANCEL_THREADS 527 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
528 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status);
530 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
531 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
534 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 536 __kmp_clear_x87_fpu_status_word();
537 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
538 __kmp_load_mxcsr(&__kmp_init_mxcsr);
541 #ifdef KMP_BLOCK_SIGNALS 542 status = sigfillset(&new_set);
543 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status);
544 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
545 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status);
548 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 549 if (__kmp_stkoffset > 0 && gtid > 0) {
550 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
555 __kmp_set_stack_info(gtid, (kmp_info_t *)thr);
557 __kmp_check_stack_overlap((kmp_info_t *)thr);
559 exit_val = __kmp_launch_thread((kmp_info_t *)thr);
561 #ifdef KMP_BLOCK_SIGNALS 562 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
563 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status);
572 static void *__kmp_launch_monitor(
void *thr) {
573 int status, old_type, old_state;
574 #ifdef KMP_BLOCK_SIGNALS 577 struct timespec interval;
579 int yield_cycles = 0;
583 KA_TRACE(10, (
"__kmp_launch_monitor: #1 launched\n"));
586 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
587 #ifdef KMP_TDATA_GTID 588 __kmp_gtid = KMP_GTID_MONITOR;
595 __kmp_itt_thread_ignore();
598 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
601 __kmp_check_stack_overlap((kmp_info_t *)thr);
603 #ifdef KMP_CANCEL_THREADS 604 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
605 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status);
607 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
608 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
611 #if KMP_REAL_TIME_FIX 616 int sched = sched_getscheduler(0);
617 if (sched == SCHED_FIFO || sched == SCHED_RR) {
620 struct sched_param param;
621 int max_priority = sched_get_priority_max(sched);
623 KMP_WARNING(RealTimeSchedNotSupported);
624 sched_getparam(0, ¶m);
625 if (param.sched_priority < max_priority) {
626 param.sched_priority += 1;
627 rc = sched_setscheduler(0, sched, ¶m);
630 kmp_msg_t err_code = KMP_ERR(error);
631 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
632 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
633 if (__kmp_generate_warnings == kmp_warnings_off) {
634 __kmp_str_free(&err_code.str);
641 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
642 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
647 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
649 #endif // KMP_REAL_TIME_FIX 653 if (__kmp_monitor_wakeups == 1) {
655 interval.tv_nsec = 0;
658 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
661 KA_TRACE(10, (
"__kmp_launch_monitor: #2 monitor\n"));
663 if (__kmp_yield_cycle) {
664 __kmp_yielding_on = 0;
665 yield_count = __kmp_yield_off_count;
667 __kmp_yielding_on = 1;
670 while (!TCR_4(__kmp_global.g.g_done)) {
676 KA_TRACE(15, (
"__kmp_launch_monitor: update\n"));
678 status = gettimeofday(&tval, NULL);
679 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
680 TIMEVAL_TO_TIMESPEC(&tval, &now);
682 now.tv_sec += interval.tv_sec;
683 now.tv_nsec += interval.tv_nsec;
685 if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
687 now.tv_nsec -= KMP_NSEC_PER_SEC;
690 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
691 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
693 if (!TCR_4(__kmp_global.g.g_done)) {
694 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
695 &__kmp_wait_mx.m_mutex, &now);
697 if (status != ETIMEDOUT && status != EINTR) {
698 KMP_SYSFAIL(
"pthread_cond_timedwait", status);
702 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
703 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
705 if (__kmp_yield_cycle) {
707 if ((yield_cycles % yield_count) == 0) {
708 if (__kmp_yielding_on) {
709 __kmp_yielding_on = 0;
710 yield_count = __kmp_yield_off_count;
712 __kmp_yielding_on = 1;
713 yield_count = __kmp_yield_on_count;
718 __kmp_yielding_on = 1;
721 TCW_4(__kmp_global.g.g_time.dt.t_value,
722 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
727 KA_TRACE(10, (
"__kmp_launch_monitor: #3 cleanup\n"));
729 #ifdef KMP_BLOCK_SIGNALS 730 status = sigfillset(&new_set);
731 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status);
732 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
733 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status);
736 KA_TRACE(10, (
"__kmp_launch_monitor: #4 finished\n"));
738 if (__kmp_global.g.g_abort != 0) {
744 KA_TRACE(10, (
"__kmp_launch_monitor: #5 terminate sig=%d\n",
745 __kmp_global.g.g_abort));
750 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
751 __kmp_terminate_thread(gtid);
755 KA_TRACE(10, (
"__kmp_launch_monitor: #6 raise sig=%d\n",
756 __kmp_global.g.g_abort));
758 if (__kmp_global.g.g_abort > 0)
759 raise(__kmp_global.g.g_abort);
762 KA_TRACE(10, (
"__kmp_launch_monitor: #7 exit\n"));
766 #endif // KMP_USE_MONITOR 768 void __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size) {
770 pthread_attr_t thread_attr;
773 th->th.th_info.ds.ds_gtid = gtid;
775 #if KMP_STATS_ENABLED 777 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
783 if (!KMP_UBER_GTID(gtid)) {
784 th->th.th_stats = __kmp_stats_list->push_back(gtid);
788 th->th.th_stats = __kmp_stats_thread_ptr;
790 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
792 #endif // KMP_STATS_ENABLED 794 if (KMP_UBER_GTID(gtid)) {
795 KA_TRACE(10, (
"__kmp_create_worker: uber thread (%d)\n", gtid));
796 th->th.th_info.ds.ds_thread = pthread_self();
797 __kmp_set_stack_info(gtid, th);
798 __kmp_check_stack_overlap(th);
802 KA_TRACE(10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid));
806 #ifdef KMP_THREAD_ATTR 807 status = pthread_attr_init(&thread_attr);
809 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
811 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
813 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
823 stack_size += gtid * __kmp_stkoffset * 2;
825 KA_TRACE(10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 826 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
827 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
829 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 830 status = pthread_attr_setstacksize(&thread_attr, stack_size);
831 #ifdef KMP_BACKUP_STKSIZE 833 if (!__kmp_env_stksize) {
834 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
835 __kmp_stksize = KMP_BACKUP_STKSIZE;
836 KA_TRACE(10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, " 837 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu " 839 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
840 status = pthread_attr_setstacksize(&thread_attr, stack_size);
845 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
846 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
853 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (
void *)th);
854 if (status != 0 || !handle) {
855 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 856 if (status == EINVAL) {
857 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
858 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
860 if (status == ENOMEM) {
861 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
862 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
865 if (status == EAGAIN) {
866 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
867 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
869 KMP_SYSFAIL(
"pthread_create", status);
872 th->th.th_info.ds.ds_thread = handle;
874 #ifdef KMP_THREAD_ATTR 875 status = pthread_attr_destroy(&thread_attr);
877 kmp_msg_t err_code = KMP_ERR(status);
878 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
880 if (__kmp_generate_warnings == kmp_warnings_off) {
881 __kmp_str_free(&err_code.str);
888 KA_TRACE(10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid));
893 void __kmp_create_monitor(kmp_info_t *th) {
895 pthread_attr_t thread_attr;
898 int auto_adj_size = FALSE;
900 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
902 KA_TRACE(10, (
"__kmp_create_monitor: skipping monitor thread because of " 904 th->th.th_info.ds.ds_tid = 0;
905 th->th.th_info.ds.ds_gtid = 0;
908 KA_TRACE(10, (
"__kmp_create_monitor: try to create monitor\n"));
912 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
913 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
914 #if KMP_REAL_TIME_FIX 915 TCW_4(__kmp_global.g.g_time.dt.t_value,
918 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
919 #endif // KMP_REAL_TIME_FIX 921 #ifdef KMP_THREAD_ATTR 922 if (__kmp_monitor_stksize == 0) {
923 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
924 auto_adj_size = TRUE;
926 status = pthread_attr_init(&thread_attr);
928 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
930 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
932 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
935 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 936 status = pthread_attr_getstacksize(&thread_attr, &size);
937 KMP_CHECK_SYSFAIL(
"pthread_attr_getstacksize", status);
939 size = __kmp_sys_min_stksize;
943 if (__kmp_monitor_stksize == 0) {
944 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
946 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
947 __kmp_monitor_stksize = __kmp_sys_min_stksize;
950 KA_TRACE(10, (
"__kmp_create_monitor: default stacksize = %lu bytes," 951 "requested stacksize = %lu bytes\n",
952 size, __kmp_monitor_stksize));
957 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 958 KA_TRACE(10, (
"__kmp_create_monitor: setting stacksize = %lu bytes,",
959 __kmp_monitor_stksize));
960 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
963 __kmp_monitor_stksize *= 2;
966 kmp_msg_t err_code = KMP_ERR(status);
967 __kmp_msg(kmp_ms_warning,
968 KMP_MSG(CantSetMonitorStackSize, (
long int)__kmp_monitor_stksize),
969 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
970 if (__kmp_generate_warnings == kmp_warnings_off) {
971 __kmp_str_free(&err_code.str);
977 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (
void *)th);
980 #ifdef _POSIX_THREAD_ATTR_STACKSIZE 981 if (status == EINVAL) {
982 if (auto_adj_size && (__kmp_monitor_stksize < (
size_t)0x40000000)) {
983 __kmp_monitor_stksize *= 2;
986 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
987 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
990 if (status == ENOMEM) {
991 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
992 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
996 if (status == EAGAIN) {
997 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
998 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1000 KMP_SYSFAIL(
"pthread_create", status);
1003 th->th.th_info.ds.ds_thread = handle;
1005 #if KMP_REAL_TIME_FIX 1007 KMP_DEBUG_ASSERT(
sizeof(kmp_uint32) ==
1008 sizeof(__kmp_global.g.g_time.dt.t_value));
1009 __kmp_wait_yield_4((kmp_uint32
volatile *)&__kmp_global.g.g_time.dt.t_value,
1010 -1, &__kmp_neq_4, NULL);
1011 #endif // KMP_REAL_TIME_FIX 1013 #ifdef KMP_THREAD_ATTR 1014 status = pthread_attr_destroy(&thread_attr);
1016 kmp_msg_t err_code = KMP_ERR(status);
1017 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1019 if (__kmp_generate_warnings == kmp_warnings_off) {
1020 __kmp_str_free(&err_code.str);
1027 KA_TRACE(10, (
"__kmp_create_monitor: monitor created %#.8lx\n",
1028 th->th.th_info.ds.ds_thread));
1031 #endif // KMP_USE_MONITOR 1033 void __kmp_exit_thread(
int exit_status) {
1034 pthread_exit((
void *)(intptr_t)exit_status);
1038 void __kmp_resume_monitor();
1040 void __kmp_reap_monitor(kmp_info_t *th) {
1044 KA_TRACE(10, (
"__kmp_reap_monitor: try to reap monitor thread with handle" 1046 th->th.th_info.ds.ds_thread));
1051 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1052 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1053 KA_TRACE(10, (
"__kmp_reap_monitor: monitor did not start, returning\n"));
1063 status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1064 if (status != ESRCH) {
1065 __kmp_resume_monitor();
1067 KA_TRACE(10, (
"__kmp_reap_monitor: try to join with monitor\n"));
1068 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1069 if (exit_val != th) {
1070 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1073 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1074 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1076 KA_TRACE(10, (
"__kmp_reap_monitor: done reaping monitor thread with handle" 1078 th->th.th_info.ds.ds_thread));
1082 #endif // KMP_USE_MONITOR 1084 void __kmp_reap_worker(kmp_info_t *th) {
1091 10, (
"__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1093 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1097 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1099 if (exit_val != th) {
1100 KA_TRACE(10, (
"__kmp_reap_worker: worker T#%d did not reap properly, " 1102 th->th.th_info.ds.ds_gtid, exit_val));
1106 KA_TRACE(10, (
"__kmp_reap_worker: done reaping T#%d\n",
1107 th->th.th_info.ds.ds_gtid));
1112 #if KMP_HANDLE_SIGNALS 1114 static void __kmp_null_handler(
int signo) {
1118 static void __kmp_team_handler(
int signo) {
1119 if (__kmp_global.g.g_abort == 0) {
1122 __kmp_debug_printf(
"__kmp_team_handler: caught signal = %d\n", signo);
1137 if (__kmp_debug_buf) {
1138 __kmp_dump_debug_buffer();
1141 TCW_4(__kmp_global.g.g_abort, signo);
1143 TCW_4(__kmp_global.g.g_done, TRUE);
1148 __kmp_debug_printf(
"__kmp_team_handler: unknown signal type");
1155 static void __kmp_sigaction(
int signum,
const struct sigaction *act,
1156 struct sigaction *oldact) {
1157 int rc = sigaction(signum, act, oldact);
1158 KMP_CHECK_SYSFAIL_ERRNO(
"sigaction", rc);
1161 static void __kmp_install_one_handler(
int sig, sig_func_t handler_func,
1162 int parallel_init) {
1165 (
"__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1166 if (parallel_init) {
1167 struct sigaction new_action;
1168 struct sigaction old_action;
1169 new_action.sa_handler = handler_func;
1170 new_action.sa_flags = 0;
1171 sigfillset(&new_action.sa_mask);
1172 __kmp_sigaction(sig, &new_action, &old_action);
1173 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1174 sigaddset(&__kmp_sigset, sig);
1177 __kmp_sigaction(sig, &old_action, NULL);
1181 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]);
1186 static void __kmp_remove_one_handler(
int sig) {
1187 KB_TRACE(60, (
"__kmp_remove_one_handler( %d )\n", sig));
1188 if (sigismember(&__kmp_sigset, sig)) {
1189 struct sigaction old;
1191 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old);
1192 if ((old.sa_handler != __kmp_team_handler) &&
1193 (old.sa_handler != __kmp_null_handler)) {
1195 KB_TRACE(10, (
"__kmp_remove_one_handler: oops, not our handler, " 1196 "restoring: sig=%d\n",
1198 __kmp_sigaction(sig, &old, NULL);
1200 sigdelset(&__kmp_sigset, sig);
1205 void __kmp_install_signals(
int parallel_init) {
1206 KB_TRACE(10, (
"__kmp_install_signals( %d )\n", parallel_init));
1207 if (__kmp_handle_signals || !parallel_init) {
1210 sigemptyset(&__kmp_sigset);
1211 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init);
1212 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1213 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init);
1214 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1215 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1216 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1217 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init);
1218 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1220 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init);
1222 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1224 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init);
1229 void __kmp_remove_signals(
void) {
1231 KB_TRACE(10, (
"__kmp_remove_signals()\n"));
1232 for (sig = 1; sig < NSIG; ++sig) {
1233 __kmp_remove_one_handler(sig);
1237 #endif // KMP_HANDLE_SIGNALS 1239 void __kmp_enable(
int new_state) {
1240 #ifdef KMP_CANCEL_THREADS 1241 int status, old_state;
1242 status = pthread_setcancelstate(new_state, &old_state);
1243 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
1244 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1248 void __kmp_disable(
int *old_state) {
1249 #ifdef KMP_CANCEL_THREADS 1251 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
1252 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
1256 static void __kmp_atfork_prepare(
void) {
1257 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
1258 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1261 static void __kmp_atfork_parent(
void) {
1262 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1263 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1269 static void __kmp_atfork_child(
void) {
1270 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1277 #if KMP_AFFINITY_SUPPORTED 1281 kmp_set_thread_affinity_mask_initial();
1286 __kmp_affinity_type = affinity_none;
1288 if (__kmp_nested_proc_bind.bind_types != NULL) {
1289 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1291 #endif // OMP_40_ENABLED 1292 #endif // KMP_AFFINITY_SUPPORTED 1294 __kmp_init_runtime = FALSE;
1296 __kmp_init_monitor = 0;
1298 __kmp_init_parallel = FALSE;
1299 __kmp_init_middle = FALSE;
1300 __kmp_init_serial = FALSE;
1301 TCW_4(__kmp_init_gtid, FALSE);
1302 __kmp_init_common = FALSE;
1304 TCW_4(__kmp_init_user_locks, FALSE);
1305 #if !KMP_USE_DYNAMIC_LOCK 1306 __kmp_user_lock_table.used = 1;
1307 __kmp_user_lock_table.allocated = 0;
1308 __kmp_user_lock_table.table = NULL;
1309 __kmp_lock_blocks = NULL;
1313 TCW_4(__kmp_nth, 0);
1315 __kmp_thread_pool = NULL;
1316 __kmp_thread_pool_insert_pt = NULL;
1317 __kmp_team_pool = NULL;
1321 KA_TRACE(10, (
"__kmp_atfork_child: checking cache address list %p\n",
1322 __kmp_threadpriv_cache_list));
1324 while (__kmp_threadpriv_cache_list != NULL) {
1326 if (*__kmp_threadpriv_cache_list->addr != NULL) {
1327 KC_TRACE(50, (
"__kmp_atfork_child: zeroing cache at address %p\n",
1328 &(*__kmp_threadpriv_cache_list->addr)));
1330 *__kmp_threadpriv_cache_list->addr = NULL;
1332 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1335 __kmp_init_runtime = FALSE;
1338 __kmp_init_bootstrap_lock(&__kmp_initz_lock);
1339 __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
1340 __kmp_init_bootstrap_lock(&__kmp_console_lock);
1341 __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
1358 void __kmp_register_atfork(
void) {
1359 if (__kmp_need_register_atfork) {
1360 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent,
1361 __kmp_atfork_child);
1362 KMP_CHECK_SYSFAIL(
"pthread_atfork", status);
1363 __kmp_need_register_atfork = FALSE;
1367 void __kmp_suspend_initialize(
void) {
1369 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
1370 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status);
1371 status = pthread_condattr_init(&__kmp_suspend_cond_attr);
1372 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status);
1375 static void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1376 ANNOTATE_HAPPENS_AFTER(&th->th.th_suspend_init_count);
1377 if (th->th.th_suspend_init_count <= __kmp_fork_count) {
1381 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
1382 &__kmp_suspend_cond_attr);
1383 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status);
1384 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
1385 &__kmp_suspend_mutex_attr);
1386 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status);
1387 *(
volatile int *)&th->th.th_suspend_init_count = __kmp_fork_count + 1;
1388 ANNOTATE_HAPPENS_BEFORE(&th->th.th_suspend_init_count);
1392 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1393 if (th->th.th_suspend_init_count > __kmp_fork_count) {
1398 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond);
1399 if (status != 0 && status != EBUSY) {
1400 KMP_SYSFAIL(
"pthread_cond_destroy", status);
1402 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex);
1403 if (status != 0 && status != EBUSY) {
1404 KMP_SYSFAIL(
"pthread_mutex_destroy", status);
1406 --th->th.th_suspend_init_count;
1407 KMP_DEBUG_ASSERT(th->th.th_suspend_init_count == __kmp_fork_count);
1414 static inline void __kmp_suspend_template(
int th_gtid, C *flag) {
1415 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1416 kmp_info_t *th = __kmp_threads[th_gtid];
1418 typename C::flag_t old_spin;
1420 KF_TRACE(30, (
"__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1423 __kmp_suspend_initialize_thread(th);
1425 status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1426 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
1428 KF_TRACE(10, (
"__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1429 th_gtid, flag->get()));
1433 old_spin = flag->set_sleeping();
1435 KF_TRACE(5, (
"__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x," 1437 th_gtid, flag->get(), *(flag->get()), old_spin));
1439 if (flag->done_check_val(old_spin)) {
1440 old_spin = flag->unset_sleeping();
1441 KF_TRACE(5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit " 1443 th_gtid, flag->get()));
1448 int deactivated = FALSE;
1449 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
1451 while (flag->is_sleeping()) {
1452 #ifdef DEBUG_SUSPEND 1454 __kmp_suspend_count++;
1455 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1456 __kmp_printf(
"__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1462 th->th.th_active = FALSE;
1463 if (th->th.th_active_in_pool) {
1464 th->th.th_active_in_pool = FALSE;
1465 KMP_TEST_THEN_DEC32(&__kmp_thread_pool_active_nth);
1466 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1471 #if USE_SUSPEND_TIMEOUT 1472 struct timespec now;
1473 struct timeval tval;
1476 status = gettimeofday(&tval, NULL);
1477 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
1478 TIMEVAL_TO_TIMESPEC(&tval, &now);
1480 msecs = (4 * __kmp_dflt_blocktime) + 200;
1481 now.tv_sec += msecs / 1000;
1482 now.tv_nsec += (msecs % 1000) * 1000;
1484 KF_TRACE(15, (
"__kmp_suspend_template: T#%d about to perform " 1485 "pthread_cond_timedwait\n",
1487 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1488 &th->th.th_suspend_mx.m_mutex, &now);
1490 KF_TRACE(15, (
"__kmp_suspend_template: T#%d about to perform" 1491 " pthread_cond_wait\n",
1493 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
1494 &th->th.th_suspend_mx.m_mutex);
1497 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1498 KMP_SYSFAIL(
"pthread_cond_wait", status);
1501 if (status == ETIMEDOUT) {
1502 if (flag->is_sleeping()) {
1504 (
"__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1506 KF_TRACE(2, (
"__kmp_suspend_template: T#%d timeout wakeup, sleep bit " 1510 }
else if (flag->is_sleeping()) {
1512 (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1519 th->th.th_active = TRUE;
1520 if (TCR_4(th->th.th_in_pool)) {
1521 KMP_TEST_THEN_INC32(&__kmp_thread_pool_active_nth);
1522 th->th.th_active_in_pool = TRUE;
1526 #ifdef DEBUG_SUSPEND 1529 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1530 __kmp_printf(
"__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1535 status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1536 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
1537 KF_TRACE(30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid));
1540 void __kmp_suspend_32(
int th_gtid, kmp_flag_32 *flag) {
1541 __kmp_suspend_template(th_gtid, flag);
1543 void __kmp_suspend_64(
int th_gtid, kmp_flag_64 *flag) {
1544 __kmp_suspend_template(th_gtid, flag);
1546 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
1547 __kmp_suspend_template(th_gtid, flag);
1554 static inline void __kmp_resume_template(
int target_gtid, C *flag) {
1555 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1556 kmp_info_t *th = __kmp_threads[target_gtid];
1560 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1563 KF_TRACE(30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1564 gtid, target_gtid));
1565 KMP_DEBUG_ASSERT(gtid != target_gtid);
1567 __kmp_suspend_initialize_thread(th);
1569 status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1570 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
1573 flag = (C *)CCAST(
void *, th->th.th_sleep_loc);
1578 if (!flag || flag->get_type() != flag->get_ptr_type()) {
1581 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already " 1582 "awake: flag(%p)\n",
1583 gtid, target_gtid, NULL));
1584 status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1585 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
1589 typename C::flag_t old_spin = flag->unset_sleeping();
1590 if (!flag->is_sleeping_val(old_spin)) {
1591 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already " 1594 gtid, target_gtid, flag->get(), old_spin, *flag->get()));
1595 status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1596 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
1599 KF_TRACE(5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset " 1600 "sleep bit for flag's loc(%p): " 1602 gtid, target_gtid, flag->get(), old_spin, *flag->get()));
1604 TCW_PTR(th->th.th_sleep_loc, NULL);
1606 #ifdef DEBUG_SUSPEND 1609 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1610 __kmp_printf(
"__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1611 target_gtid, buffer);
1614 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
1615 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status);
1616 status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1617 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
1618 KF_TRACE(30, (
"__kmp_resume_template: T#%d exiting after signaling wake up" 1620 gtid, target_gtid));
1623 void __kmp_resume_32(
int target_gtid, kmp_flag_32 *flag) {
1624 __kmp_resume_template(target_gtid, flag);
1626 void __kmp_resume_64(
int target_gtid, kmp_flag_64 *flag) {
1627 __kmp_resume_template(target_gtid, flag);
1629 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
1630 __kmp_resume_template(target_gtid, flag);
1634 void __kmp_resume_monitor() {
1635 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1638 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1639 KF_TRACE(30, (
"__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1641 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1643 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1644 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
1645 #ifdef DEBUG_SUSPEND 1648 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1649 __kmp_printf(
"__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1650 KMP_GTID_MONITOR, buffer);
1653 status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1654 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status);
1655 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1656 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
1657 KF_TRACE(30, (
"__kmp_resume_monitor: T#%d exiting after signaling wake up" 1659 gtid, KMP_GTID_MONITOR));
1661 #endif // KMP_USE_MONITOR 1663 void __kmp_yield(
int cond) {
1667 if (!__kmp_yielding_on)
1670 if (__kmp_yield_cycle && !KMP_YIELD_NOW())
1676 void __kmp_gtid_set_specific(
int gtid) {
1677 if (__kmp_init_gtid) {
1679 status = pthread_setspecific(__kmp_gtid_threadprivate_key,
1680 (
void *)(intptr_t)(gtid + 1));
1681 KMP_CHECK_SYSFAIL(
"pthread_setspecific", status);
1683 KA_TRACE(50, (
"__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1687 int __kmp_gtid_get_specific() {
1689 if (!__kmp_init_gtid) {
1690 KA_TRACE(50, (
"__kmp_gtid_get_specific: runtime shutdown, returning " 1691 "KMP_GTID_SHUTDOWN\n"));
1692 return KMP_GTID_SHUTDOWN;
1694 gtid = (int)(
size_t)pthread_getspecific(__kmp_gtid_threadprivate_key);
1696 gtid = KMP_GTID_DNE;
1700 KA_TRACE(50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
1701 __kmp_gtid_threadprivate_key, gtid));
1705 double __kmp_read_cpu_time(
void) {
1711 return (buffer.tms_utime + buffer.tms_cutime) / (double)CLOCKS_PER_SEC;
1714 int __kmp_read_system_info(
struct kmp_sys_info *info) {
1716 struct rusage r_usage;
1718 memset(info, 0,
sizeof(*info));
1720 status = getrusage(RUSAGE_SELF, &r_usage);
1721 KMP_CHECK_SYSFAIL_ERRNO(
"getrusage", status);
1724 info->maxrss = r_usage.ru_maxrss;
1726 info->minflt = r_usage.ru_minflt;
1728 info->majflt = r_usage.ru_majflt;
1730 info->nswap = r_usage.ru_nswap;
1732 info->inblock = r_usage.ru_inblock;
1734 info->oublock = r_usage.ru_oublock;
1736 info->nvcsw = r_usage.ru_nvcsw;
1738 info->nivcsw = r_usage.ru_nivcsw;
1740 return (status != 0);
1743 void __kmp_read_system_time(
double *delta) {
1745 struct timeval tval;
1746 struct timespec stop;
1749 status = gettimeofday(&tval, NULL);
1750 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
1751 TIMEVAL_TO_TIMESPEC(&tval, &stop);
1752 t_ns = TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start);
1753 *delta = (t_ns * 1e-9);
1756 void __kmp_clear_system_time(
void) {
1757 struct timeval tval;
1759 status = gettimeofday(&tval, NULL);
1760 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
1761 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1764 static int __kmp_get_xproc(
void) {
1768 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD 1770 r = sysconf(_SC_NPROCESSORS_ONLN);
1778 host_basic_info_data_t info;
1779 mach_msg_type_number_t num = HOST_BASIC_INFO_COUNT;
1780 rc = host_info(mach_host_self(), HOST_BASIC_INFO, (host_info_t)&info, &num);
1781 if (rc == 0 && num == HOST_BASIC_INFO_COUNT) {
1784 r = info.avail_cpus;
1786 KMP_WARNING(CantGetNumAvailCPU);
1787 KMP_INFORM(AssumedNumCPU);
1792 #error "Unknown or unsupported OS." 1796 return r > 0 ? r : 2;
1800 int __kmp_read_from_file(
char const *path,
char const *format, ...) {
1804 va_start(args, format);
1805 FILE *f = fopen(path,
"rb");
1808 result = vfscanf(f, format, args);
1814 void __kmp_runtime_initialize(
void) {
1816 pthread_mutexattr_t mutex_attr;
1817 pthread_condattr_t cond_attr;
1819 if (__kmp_init_runtime) {
1823 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64) 1824 if (!__kmp_cpuinfo.initialized) {
1825 __kmp_query_cpuid(&__kmp_cpuinfo);
1829 __kmp_xproc = __kmp_get_xproc();
1831 if (sysconf(_SC_THREADS)) {
1834 __kmp_sys_max_nth = sysconf(_SC_THREAD_THREADS_MAX);
1835 if (__kmp_sys_max_nth == -1) {
1837 __kmp_sys_max_nth = INT_MAX;
1838 }
else if (__kmp_sys_max_nth <= 1) {
1840 __kmp_sys_max_nth = KMP_MAX_NTH;
1844 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1845 if (__kmp_sys_min_stksize <= 1) {
1846 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1851 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
1853 status = pthread_key_create(&__kmp_gtid_threadprivate_key,
1854 __kmp_internal_end_dest);
1855 KMP_CHECK_SYSFAIL(
"pthread_key_create", status);
1856 status = pthread_mutexattr_init(&mutex_attr);
1857 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status);
1858 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr);
1859 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status);
1860 status = pthread_condattr_init(&cond_attr);
1861 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status);
1862 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr);
1863 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status);
1865 __kmp_itt_initialize();
1868 __kmp_init_runtime = TRUE;
1871 void __kmp_runtime_destroy(
void) {
1874 if (!__kmp_init_runtime) {
1879 __kmp_itt_destroy();
1882 status = pthread_key_delete(__kmp_gtid_threadprivate_key);
1883 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status);
1885 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex);
1886 if (status != 0 && status != EBUSY) {
1887 KMP_SYSFAIL(
"pthread_mutex_destroy", status);
1889 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond);
1890 if (status != 0 && status != EBUSY) {
1891 KMP_SYSFAIL(
"pthread_cond_destroy", status);
1893 #if KMP_AFFINITY_SUPPORTED 1894 __kmp_affinity_uninitialize();
1897 __kmp_init_runtime = FALSE;
1902 void __kmp_thread_sleep(
int millis) { sleep((millis + 500) / 1000); }
1905 void __kmp_elapsed(
double *t) {
1907 #ifdef FIX_SGI_CLOCK 1910 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
1911 KMP_CHECK_SYSFAIL_ERRNO(
"clock_gettime", status);
1913 (double)ts.tv_nsec * (1.0 / (
double)KMP_NSEC_PER_SEC) + (
double)ts.tv_sec;
1917 status = gettimeofday(&tv, NULL);
1918 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
1920 (double)tv.tv_usec * (1.0 / (
double)KMP_USEC_PER_SEC) + (
double)tv.tv_sec;
1925 void __kmp_elapsed_tick(
double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
1928 kmp_uint64 __kmp_now_nsec() {
1930 gettimeofday(&t, NULL);
1931 return KMP_NSEC_PER_SEC * t.tv_sec + 1000 * t.tv_usec;
1934 #if KMP_ARCH_X86 || KMP_ARCH_X86_64 1936 void __kmp_initialize_system_tick() {
1937 kmp_uint64 delay = 100000;
1938 kmp_uint64 nsec = __kmp_now_nsec();
1939 kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
1941 while ((now = __kmp_hardware_timestamp()) < goal)
1943 __kmp_ticks_per_msec =
1944 (kmp_uint64)(1e6 * (delay + (now - goal)) / (__kmp_now_nsec() - nsec));
1951 int __kmp_is_address_mapped(
void *addr) {
1956 #if KMP_OS_LINUX || KMP_OS_FREEBSD 1961 char *name = __kmp_str_format(
"/proc/%d/maps", getpid());
1964 file = fopen(name,
"r");
1965 KMP_ASSERT(file != NULL);
1969 void *beginning = NULL;
1970 void *ending = NULL;
1973 rc = fscanf(file,
"%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
1977 KMP_ASSERT(rc == 3 &&
1978 KMP_STRLEN(perms) == 4);
1981 if ((addr >= beginning) && (addr < ending)) {
1983 if (strcmp(perms,
"rw") == 0) {
1993 KMP_INTERNAL_FREE(name);
2002 rc = vm_read_overwrite(
2004 (vm_address_t)(addr),
2006 (vm_address_t)(&buffer),
2014 #elif KMP_OS_FREEBSD || KMP_OS_NETBSD 2021 #error "Unknown or unsupported OS" 2029 #ifdef USE_LOAD_BALANCE 2038 int __kmp_get_load_balance(
int max) {
2042 int res = getloadavg(averages, 3);
2047 if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2048 ret_avg = averages[0];
2049 }
else if ((__kmp_load_balance_interval >= 180 &&
2050 __kmp_load_balance_interval < 600) &&
2052 ret_avg = averages[1];
2053 }
else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2054 ret_avg = averages[2];
2068 int __kmp_get_load_balance(
int max) {
2069 static int permanent_error = 0;
2070 static int glb_running_threads = 0;
2072 static double glb_call_time = 0;
2074 int running_threads = 0;
2076 DIR *proc_dir = NULL;
2077 struct dirent *proc_entry = NULL;
2079 kmp_str_buf_t task_path;
2080 DIR *task_dir = NULL;
2081 struct dirent *task_entry = NULL;
2082 int task_path_fixed_len;
2084 kmp_str_buf_t stat_path;
2086 int stat_path_fixed_len;
2088 int total_processes = 0;
2089 int total_threads = 0;
2091 double call_time = 0.0;
2093 __kmp_str_buf_init(&task_path);
2094 __kmp_str_buf_init(&stat_path);
2096 __kmp_elapsed(&call_time);
2098 if (glb_call_time &&
2099 (call_time - glb_call_time < __kmp_load_balance_interval)) {
2100 running_threads = glb_running_threads;
2104 glb_call_time = call_time;
2107 if (permanent_error) {
2108 running_threads = -1;
2117 proc_dir = opendir(
"/proc");
2118 if (proc_dir == NULL) {
2121 running_threads = -1;
2122 permanent_error = 1;
2127 __kmp_str_buf_cat(&task_path,
"/proc/", 6);
2128 task_path_fixed_len = task_path.used;
2130 proc_entry = readdir(proc_dir);
2131 while (proc_entry != NULL) {
2134 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2142 KMP_DEBUG_ASSERT(total_processes != 1 ||
2143 strcmp(proc_entry->d_name,
"1") == 0);
2146 task_path.used = task_path_fixed_len;
2147 __kmp_str_buf_cat(&task_path, proc_entry->d_name,
2148 KMP_STRLEN(proc_entry->d_name));
2149 __kmp_str_buf_cat(&task_path,
"/task", 5);
2151 task_dir = opendir(task_path.str);
2152 if (task_dir == NULL) {
2161 if (strcmp(proc_entry->d_name,
"1") == 0) {
2162 running_threads = -1;
2163 permanent_error = 1;
2168 __kmp_str_buf_clear(&stat_path);
2169 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used);
2170 __kmp_str_buf_cat(&stat_path,
"/", 1);
2171 stat_path_fixed_len = stat_path.used;
2173 task_entry = readdir(task_dir);
2174 while (task_entry != NULL) {
2176 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2184 stat_path_fixed_len;
2185 __kmp_str_buf_cat(&stat_path, task_entry->d_name,
2186 KMP_STRLEN(task_entry->d_name));
2187 __kmp_str_buf_cat(&stat_path,
"/stat", 5);
2191 stat_file = open(stat_path.str, O_RDONLY);
2192 if (stat_file == -1) {
2222 len = read(stat_file, buffer,
sizeof(buffer) - 1);
2229 char *close_parent = strstr(buffer,
") ");
2230 if (close_parent != NULL) {
2231 char state = *(close_parent + 2);
2234 if (running_threads >= max) {
2244 task_entry = readdir(task_dir);
2250 proc_entry = readdir(proc_dir);
2256 KMP_DEBUG_ASSERT(running_threads > 0);
2257 if (running_threads <= 0) {
2258 running_threads = 1;
2262 if (proc_dir != NULL) {
2265 __kmp_str_buf_free(&task_path);
2266 if (task_dir != NULL) {
2269 __kmp_str_buf_free(&stat_path);
2270 if (stat_file != -1) {
2274 glb_running_threads = running_threads;
2276 return running_threads;
2280 #endif // KMP_OS_DARWIN 2282 #endif // USE_LOAD_BALANCE 2284 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \ 2285 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || KMP_ARCH_PPC64) 2289 int __kmp_invoke_microtask(microtask_t pkfn,
int gtid,
int tid,
int argc,
2293 void **exit_frame_ptr
2297 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2302 fprintf(stderr,
"Too many args to microtask: %d!\n", argc);
2306 (*pkfn)(>id, &tid);
2309 (*pkfn)(>id, &tid, p_argv[0]);
2312 (*pkfn)(>id, &tid, p_argv[0], p_argv[1]);
2315 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]);
2318 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3]);
2321 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4]);
2324 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2328 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2329 p_argv[5], p_argv[6]);
2332 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2333 p_argv[5], p_argv[6], p_argv[7]);
2336 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2337 p_argv[5], p_argv[6], p_argv[7], p_argv[8]);
2340 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2341 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9]);
2344 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2345 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2348 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2349 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2353 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2354 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2355 p_argv[11], p_argv[12]);
2358 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2359 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2360 p_argv[11], p_argv[12], p_argv[13]);
2363 (*pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2], p_argv[3], p_argv[4],
2364 p_argv[5], p_argv[6], p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2365 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2370 *exit_frame_ptr = 0;
#define KMP_START_EXPLICIT_TIMER(name)
"Starts" an explicit timer which will need a corresponding KMP_STOP_EXPLICIT_TIMER() macro...
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the paritioned timers to begin with name.