LLVM OpenMP* Runtime Library
kmp_tasking.cpp
1 /*
2  * kmp_tasking.cpp -- OpenMP 3.0 tasking support.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // The LLVM Compiler Infrastructure
8 //
9 // This file is dual licensed under the MIT and the University of Illinois Open
10 // Source Licenses. See LICENSE.txt for details.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "kmp.h"
15 #include "kmp_i18n.h"
16 #include "kmp_itt.h"
17 #include "kmp_stats.h"
18 #include "kmp_wait_release.h"
19 
20 #if OMPT_SUPPORT
21 #include "ompt-specific.h"
22 #endif
23 
24 #include "tsan_annotations.h"
25 
26 /* forward declaration */
27 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
28  kmp_info_t *this_thr);
29 static void __kmp_alloc_task_deque(kmp_info_t *thread,
30  kmp_thread_data_t *thread_data);
31 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
32  kmp_task_team_t *task_team);
33 
34 #ifdef OMP_45_ENABLED
35 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
36 #endif
37 
38 #ifdef BUILD_TIED_TASK_STACK
39 
40 // __kmp_trace_task_stack: print the tied tasks from the task stack in order
41 // from top do bottom
42 //
43 // gtid: global thread identifier for thread containing stack
44 // thread_data: thread data for task team thread containing stack
45 // threshold: value above which the trace statement triggers
46 // location: string identifying call site of this function (for trace)
47 static void __kmp_trace_task_stack(kmp_int32 gtid,
48  kmp_thread_data_t *thread_data,
49  int threshold, char *location) {
50  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
51  kmp_taskdata_t **stack_top = task_stack->ts_top;
52  kmp_int32 entries = task_stack->ts_entries;
53  kmp_taskdata_t *tied_task;
54 
55  KA_TRACE(
56  threshold,
57  ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
58  "first_block = %p, stack_top = %p \n",
59  location, gtid, entries, task_stack->ts_first_block, stack_top));
60 
61  KMP_DEBUG_ASSERT(stack_top != NULL);
62  KMP_DEBUG_ASSERT(entries > 0);
63 
64  while (entries != 0) {
65  KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0]);
66  // fix up ts_top if we need to pop from previous block
67  if (entries & TASK_STACK_INDEX_MASK == 0) {
68  kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top);
69 
70  stack_block = stack_block->sb_prev;
71  stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
72  }
73 
74  // finish bookkeeping
75  stack_top--;
76  entries--;
77 
78  tied_task = *stack_top;
79 
80  KMP_DEBUG_ASSERT(tied_task != NULL);
81  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
82 
83  KA_TRACE(threshold,
84  ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
85  "stack_top=%p, tied_task=%p\n",
86  location, gtid, entries, stack_top, tied_task));
87  }
88  KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0]);
89 
90  KA_TRACE(threshold,
91  ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n",
92  location, gtid));
93 }
94 
95 // __kmp_init_task_stack: initialize the task stack for the first time
96 // after a thread_data structure is created.
97 // It should not be necessary to do this again (assuming the stack works).
98 //
99 // gtid: global thread identifier of calling thread
100 // thread_data: thread data for task team thread containing stack
101 static void __kmp_init_task_stack(kmp_int32 gtid,
102  kmp_thread_data_t *thread_data) {
103  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
104  kmp_stack_block_t *first_block;
105 
106  // set up the first block of the stack
107  first_block = &task_stack->ts_first_block;
108  task_stack->ts_top = (kmp_taskdata_t **)first_block;
109  memset((void *)first_block, '\0',
110  TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *));
111 
112  // initialize the stack to be empty
113  task_stack->ts_entries = TASK_STACK_EMPTY;
114  first_block->sb_next = NULL;
115  first_block->sb_prev = NULL;
116 }
117 
118 // __kmp_free_task_stack: free the task stack when thread_data is destroyed.
119 //
120 // gtid: global thread identifier for calling thread
121 // thread_data: thread info for thread containing stack
122 static void __kmp_free_task_stack(kmp_int32 gtid,
123  kmp_thread_data_t *thread_data) {
124  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
125  kmp_stack_block_t *stack_block = &task_stack->ts_first_block;
126 
127  KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY);
128  // free from the second block of the stack
129  while (stack_block != NULL) {
130  kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL;
131 
132  stack_block->sb_next = NULL;
133  stack_block->sb_prev = NULL;
134  if (stack_block != &task_stack->ts_first_block) {
135  __kmp_thread_free(thread,
136  stack_block); // free the block, if not the first
137  }
138  stack_block = next_block;
139  }
140  // initialize the stack to be empty
141  task_stack->ts_entries = 0;
142  task_stack->ts_top = NULL;
143 }
144 
145 // __kmp_push_task_stack: Push the tied task onto the task stack.
146 // Grow the stack if necessary by allocating another block.
147 //
148 // gtid: global thread identifier for calling thread
149 // thread: thread info for thread containing stack
150 // tied_task: the task to push on the stack
151 static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread,
152  kmp_taskdata_t *tied_task) {
153  // GEH - need to consider what to do if tt_threads_data not allocated yet
154  kmp_thread_data_t *thread_data =
155  &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)];
156  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
157 
158  if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) {
159  return; // Don't push anything on stack if team or team tasks are serialized
160  }
161 
162  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
163  KMP_DEBUG_ASSERT(task_stack->ts_top != NULL);
164 
165  KA_TRACE(20,
166  ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n",
167  gtid, thread, tied_task));
168  // Store entry
169  *(task_stack->ts_top) = tied_task;
170 
171  // Do bookkeeping for next push
172  task_stack->ts_top++;
173  task_stack->ts_entries++;
174 
175  if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
176  // Find beginning of this task block
177  kmp_stack_block_t *stack_block =
178  (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE);
179 
180  // Check if we already have a block
181  if (stack_block->sb_next !=
182  NULL) { // reset ts_top to beginning of next block
183  task_stack->ts_top = &stack_block->sb_next->sb_block[0];
184  } else { // Alloc new block and link it up
185  kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc(
186  thread, sizeof(kmp_stack_block_t));
187 
188  task_stack->ts_top = &new_block->sb_block[0];
189  stack_block->sb_next = new_block;
190  new_block->sb_prev = stack_block;
191  new_block->sb_next = NULL;
192 
193  KA_TRACE(
194  30,
195  ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n",
196  gtid, tied_task, new_block));
197  }
198  }
199  KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid,
200  tied_task));
201 }
202 
203 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return
204 // the task, just check to make sure it matches the ending task passed in.
205 //
206 // gtid: global thread identifier for the calling thread
207 // thread: thread info structure containing stack
208 // tied_task: the task popped off the stack
209 // ending_task: the task that is ending (should match popped task)
210 static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread,
211  kmp_taskdata_t *ending_task) {
212  // GEH - need to consider what to do if tt_threads_data not allocated yet
213  kmp_thread_data_t *thread_data =
214  &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)];
215  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
216  kmp_taskdata_t *tied_task;
217 
218  if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) {
219  // Don't pop anything from stack if team or team tasks are serialized
220  return;
221  }
222 
223  KMP_DEBUG_ASSERT(task_stack->ts_top != NULL);
224  KMP_DEBUG_ASSERT(task_stack->ts_entries > 0);
225 
226  KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid,
227  thread));
228 
229  // fix up ts_top if we need to pop from previous block
230  if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
231  kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top);
232 
233  stack_block = stack_block->sb_prev;
234  task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
235  }
236 
237  // finish bookkeeping
238  task_stack->ts_top--;
239  task_stack->ts_entries--;
240 
241  tied_task = *(task_stack->ts_top);
242 
243  KMP_DEBUG_ASSERT(tied_task != NULL);
244  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
245  KMP_DEBUG_ASSERT(tied_task == ending_task); // If we built the stack correctly
246 
247  KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid,
248  tied_task));
249  return;
250 }
251 #endif /* BUILD_TIED_TASK_STACK */
252 
253 // __kmp_push_task: Add a task to the thread's deque
254 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
255  kmp_info_t *thread = __kmp_threads[gtid];
256  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
257  kmp_task_team_t *task_team = thread->th.th_task_team;
258  kmp_int32 tid = __kmp_tid_from_gtid(gtid);
259  kmp_thread_data_t *thread_data;
260 
261  KA_TRACE(20,
262  ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata));
263 
264  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
265  // untied task needs to increment counter so that the task structure is not
266  // freed prematurely
267  kmp_int32 counter = 1 + KMP_TEST_THEN_INC32(&taskdata->td_untied_count);
268  KA_TRACE(
269  20,
270  ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n",
271  gtid, counter, taskdata));
272  }
273 
274  // The first check avoids building task_team thread data if serialized
275  if (taskdata->td_flags.task_serial) {
276  KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning "
277  "TASK_NOT_PUSHED for task %p\n",
278  gtid, taskdata));
279  return TASK_NOT_PUSHED;
280  }
281 
282  // Now that serialized tasks have returned, we can assume that we are not in
283  // immediate exec mode
284  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
285  if (!KMP_TASKING_ENABLED(task_team)) {
286  __kmp_enable_tasking(task_team, thread);
287  }
288  KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE);
289  KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL);
290 
291  // Find tasking deque specific to encountering thread
292  thread_data = &task_team->tt.tt_threads_data[tid];
293 
294  // No lock needed since only owner can allocate
295  if (thread_data->td.td_deque == NULL) {
296  __kmp_alloc_task_deque(thread, thread_data);
297  }
298 
299  // Check if deque is full
300  if (TCR_4(thread_data->td.td_deque_ntasks) >=
301  TASK_DEQUE_SIZE(thread_data->td)) {
302  KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning "
303  "TASK_NOT_PUSHED for task %p\n",
304  gtid, taskdata));
305  return TASK_NOT_PUSHED;
306  }
307 
308  // Lock the deque for the task push operation
309  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
310 
311 #if OMP_45_ENABLED
312  // Need to recheck as we can get a proxy task from a thread outside of OpenMP
313  if (TCR_4(thread_data->td.td_deque_ntasks) >=
314  TASK_DEQUE_SIZE(thread_data->td)) {
315  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
316  KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; returning "
317  "TASK_NOT_PUSHED for task %p\n",
318  gtid, taskdata));
319  return TASK_NOT_PUSHED;
320  }
321 #else
322  // Must have room since no thread can add tasks but calling thread
323  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) <
324  TASK_DEQUE_SIZE(thread_data->td));
325 #endif
326 
327  thread_data->td.td_deque[thread_data->td.td_deque_tail] =
328  taskdata; // Push taskdata
329  // Wrap index.
330  thread_data->td.td_deque_tail =
331  (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
332  TCW_4(thread_data->td.td_deque_ntasks,
333  TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count
334 
335  KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
336  "task=%p ntasks=%d head=%u tail=%u\n",
337  gtid, taskdata, thread_data->td.td_deque_ntasks,
338  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
339 
340  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
341 
342  return TASK_SUCCESSFULLY_PUSHED;
343 }
344 
345 // __kmp_pop_current_task_from_thread: set up current task from called thread
346 // when team ends
347 //
348 // this_thr: thread structure to set current_task in.
349 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) {
350  KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d "
351  "this_thread=%p, curtask=%p, "
352  "curtask_parent=%p\n",
353  0, this_thr, this_thr->th.th_current_task,
354  this_thr->th.th_current_task->td_parent));
355 
356  this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent;
357 
358  KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d "
359  "this_thread=%p, curtask=%p, "
360  "curtask_parent=%p\n",
361  0, this_thr, this_thr->th.th_current_task,
362  this_thr->th.th_current_task->td_parent));
363 }
364 
365 // __kmp_push_current_task_to_thread: set up current task in called thread for a
366 // new team
367 //
368 // this_thr: thread structure to set up
369 // team: team for implicit task data
370 // tid: thread within team to set up
371 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team,
372  int tid) {
373  // current task of the thread is a parent of the new just created implicit
374  // tasks of new team
375  KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
376  "curtask=%p "
377  "parent_task=%p\n",
378  tid, this_thr, this_thr->th.th_current_task,
379  team->t.t_implicit_task_taskdata[tid].td_parent));
380 
381  KMP_DEBUG_ASSERT(this_thr != NULL);
382 
383  if (tid == 0) {
384  if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) {
385  team->t.t_implicit_task_taskdata[0].td_parent =
386  this_thr->th.th_current_task;
387  this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0];
388  }
389  } else {
390  team->t.t_implicit_task_taskdata[tid].td_parent =
391  team->t.t_implicit_task_taskdata[0].td_parent;
392  this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid];
393  }
394 
395  KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
396  "curtask=%p "
397  "parent_task=%p\n",
398  tid, this_thr, this_thr->th.th_current_task,
399  team->t.t_implicit_task_taskdata[tid].td_parent));
400 }
401 
402 // __kmp_task_start: bookkeeping for a task starting execution
403 //
404 // GTID: global thread id of calling thread
405 // task: task starting execution
406 // current_task: task suspending
407 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task,
408  kmp_taskdata_t *current_task) {
409  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
410  kmp_info_t *thread = __kmp_threads[gtid];
411 
412  KA_TRACE(10,
413  ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
414  gtid, taskdata, current_task));
415 
416  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
417 
418  // mark currently executing task as suspended
419  // TODO: GEH - make sure root team implicit task is initialized properly.
420  // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 );
421  current_task->td_flags.executing = 0;
422 
423 // Add task to stack if tied
424 #ifdef BUILD_TIED_TASK_STACK
425  if (taskdata->td_flags.tiedness == TASK_TIED) {
426  __kmp_push_task_stack(gtid, thread, taskdata);
427  }
428 #endif /* BUILD_TIED_TASK_STACK */
429 
430  // mark starting task as executing and as current task
431  thread->th.th_current_task = taskdata;
432 
433  KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 ||
434  taskdata->td_flags.tiedness == TASK_UNTIED);
435  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 ||
436  taskdata->td_flags.tiedness == TASK_UNTIED);
437  taskdata->td_flags.started = 1;
438  taskdata->td_flags.executing = 1;
439  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
440  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
441 
442  // GEH TODO: shouldn't we pass some sort of location identifier here?
443  // APT: yes, we will pass location here.
444  // need to store current thread state (in a thread or taskdata structure)
445  // before setting work_state, otherwise wrong state is set after end of task
446 
447  KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata));
448 
449  return;
450 }
451 
452 #if OMPT_SUPPORT
453 //------------------------------------------------------------------------------
454 // __ompt_task_init:
455 // Initialize OMPT fields maintained by a task. This will only be called after
456 // ompt_start_tool, so we already know whether ompt is enabled or not.
457 
458 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) {
459  // The calls to __ompt_task_init already have the ompt_enabled condition.
460  task->ompt_task_info.task_data.value = 0;
461  task->ompt_task_info.frame.exit_frame = NULL;
462  task->ompt_task_info.frame.enter_frame = NULL;
463 #if OMP_40_ENABLED
464  task->ompt_task_info.ndeps = 0;
465  task->ompt_task_info.deps = NULL;
466 #endif /* OMP_40_ENABLED */
467 }
468 
469 // __ompt_task_start:
470 // Build and trigger task-begin event
471 static inline void __ompt_task_start(kmp_task_t *task,
472  kmp_taskdata_t *current_task,
473  kmp_int32 gtid) {
474  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
475  ompt_task_status_t status = ompt_task_others;
476  if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) {
477  status = ompt_task_yield;
478  __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0;
479  }
480  /* let OMPT know that we're about to run this task */
481  if (ompt_enabled.ompt_callback_task_schedule) {
482  ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
483  &(current_task->ompt_task_info.task_data), status,
484  &(taskdata->ompt_task_info.task_data));
485  }
486  taskdata->ompt_task_info.scheduling_parent = current_task;
487 }
488 
489 // __ompt_task_finish:
490 // Build and trigger final task-schedule event
491 static inline void __ompt_task_finish(kmp_task_t *task,
492  kmp_taskdata_t *resumed_task) {
493  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
494  ompt_task_status_t status = ompt_task_complete;
495  if (taskdata->td_flags.tiedness == TASK_UNTIED &&
496  KMP_TEST_THEN_ADD32(&(taskdata->td_untied_count), 0) > 1)
497  status = ompt_task_others;
498  if (__kmp_omp_cancellation && taskdata->td_taskgroup &&
499  taskdata->td_taskgroup->cancel_request == cancel_taskgroup) {
500  status = ompt_task_cancel;
501  }
502 
503  /* let OMPT know that we're returning to the callee task */
504  if (ompt_enabled.ompt_callback_task_schedule) {
505  ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
506  &(taskdata->ompt_task_info.task_data), status,
507  &((resumed_task ? resumed_task
508  : (taskdata->ompt_task_info.scheduling_parent
509  ? taskdata->ompt_task_info.scheduling_parent
510  : taskdata->td_parent))
511  ->ompt_task_info.task_data));
512  }
513 }
514 #endif
515 
516 template <bool ompt>
517 static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid,
518  kmp_task_t *task,
519  void *frame_address,
520  void *return_address) {
521  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
522  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
523 
524  KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
525  "current_task=%p\n",
526  gtid, loc_ref, taskdata, current_task));
527 
528  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
529  // untied task needs to increment counter so that the task structure is not
530  // freed prematurely
531  kmp_int32 counter = 1 + KMP_TEST_THEN_INC32(&taskdata->td_untied_count);
532  KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "
533  "incremented for task %p\n",
534  gtid, counter, taskdata));
535  }
536 
537  taskdata->td_flags.task_serial =
538  1; // Execute this task immediately, not deferred.
539  __kmp_task_start(gtid, task, current_task);
540 
541 #if OMPT_SUPPORT
542  if (ompt) {
543  if (current_task->ompt_task_info.frame.enter_frame == NULL) {
544  current_task->ompt_task_info.frame.enter_frame =
545  taskdata->ompt_task_info.frame.exit_frame = frame_address;
546  }
547  if (ompt_enabled.ompt_callback_task_create) {
548  ompt_task_info_t *parent_info = &(current_task->ompt_task_info);
549  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
550  &(parent_info->task_data), &(parent_info->frame),
551  &(taskdata->ompt_task_info.task_data),
552  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0,
553  return_address);
554  }
555  __ompt_task_start(task, current_task, gtid);
556  }
557 #endif // OMPT_SUPPORT
558 
559  KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid,
560  loc_ref, taskdata));
561 }
562 
563 #if OMPT_SUPPORT
564 OMPT_NOINLINE
565 static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
566  kmp_task_t *task,
567  void *frame_address,
568  void *return_address) {
569  __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address,
570  return_address);
571 }
572 #endif // OMPT_SUPPORT
573 
574 // __kmpc_omp_task_begin_if0: report that a given serialized task has started
575 // execution
576 //
577 // loc_ref: source location information; points to beginning of task block.
578 // gtid: global thread number.
579 // task: task thunk for the started task.
580 void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid,
581  kmp_task_t *task) {
582 #if OMPT_SUPPORT
583  if (UNLIKELY(ompt_enabled.enabled)) {
584  OMPT_STORE_RETURN_ADDRESS(gtid);
585  __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task,
586  OMPT_GET_FRAME_ADDRESS(1),
587  OMPT_LOAD_RETURN_ADDRESS(gtid));
588  return;
589  }
590 #endif
591  __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL);
592 }
593 
594 #ifdef TASK_UNUSED
595 // __kmpc_omp_task_begin: report that a given task has started execution
596 // NEVER GENERATED BY COMPILER, DEPRECATED!!!
597 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) {
598  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
599 
600  KA_TRACE(
601  10,
602  ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
603  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task));
604 
605  __kmp_task_start(gtid, task, current_task);
606 
607  KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid,
608  loc_ref, KMP_TASK_TO_TASKDATA(task)));
609  return;
610 }
611 #endif // TASK_UNUSED
612 
613 // __kmp_free_task: free the current task space and the space for shareds
614 //
615 // gtid: Global thread ID of calling thread
616 // taskdata: task to free
617 // thread: thread data structure of caller
618 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
619  kmp_info_t *thread) {
620  KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid,
621  taskdata));
622 
623  // Check to make sure all flags and counters have the correct values
624  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
625  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0);
626  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1);
627  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
628  KMP_DEBUG_ASSERT(TCR_4(taskdata->td_allocated_child_tasks) == 0 ||
629  taskdata->td_flags.task_serial == 1);
630  KMP_DEBUG_ASSERT(TCR_4(taskdata->td_incomplete_child_tasks) == 0);
631 
632  taskdata->td_flags.freed = 1;
633  ANNOTATE_HAPPENS_BEFORE(taskdata);
634 // deallocate the taskdata and shared variable blocks associated with this task
635 #if USE_FAST_MEMORY
636  __kmp_fast_free(thread, taskdata);
637 #else /* ! USE_FAST_MEMORY */
638  __kmp_thread_free(thread, taskdata);
639 #endif
640 
641  KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
642 }
643 
644 // __kmp_free_task_and_ancestors: free the current task and ancestors without
645 // children
646 //
647 // gtid: Global thread ID of calling thread
648 // taskdata: task to free
649 // thread: thread data structure of caller
650 static void __kmp_free_task_and_ancestors(kmp_int32 gtid,
651  kmp_taskdata_t *taskdata,
652  kmp_info_t *thread) {
653 #if OMP_45_ENABLED
654  // Proxy tasks must always be allowed to free their parents
655  // because they can be run in background even in serial mode.
656  kmp_int32 team_serial =
657  (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) &&
658  !taskdata->td_flags.proxy;
659 #else
660  kmp_int32 team_serial =
661  taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser;
662 #endif
663  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
664 
665  kmp_int32 children =
666  KMP_TEST_THEN_DEC32(&taskdata->td_allocated_child_tasks) - 1;
667  KMP_DEBUG_ASSERT(children >= 0);
668 
669  // Now, go up the ancestor tree to see if any ancestors can now be freed.
670  while (children == 0) {
671  kmp_taskdata_t *parent_taskdata = taskdata->td_parent;
672 
673  KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
674  "and freeing itself\n",
675  gtid, taskdata));
676 
677  // --- Deallocate my ancestor task ---
678  __kmp_free_task(gtid, taskdata, thread);
679 
680  taskdata = parent_taskdata;
681 
682  // Stop checking ancestors at implicit task instead of walking up ancestor
683  // tree to avoid premature deallocation of ancestors.
684  if (team_serial || taskdata->td_flags.tasktype == TASK_IMPLICIT)
685  return;
686 
687  // Predecrement simulated by "- 1" calculation
688  children = KMP_TEST_THEN_DEC32(&taskdata->td_allocated_child_tasks) - 1;
689  KMP_DEBUG_ASSERT(children >= 0);
690  }
691 
692  KA_TRACE(
693  20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
694  "not freeing it yet\n",
695  gtid, taskdata, children));
696 }
697 
698 // __kmp_task_finish: bookkeeping to do when a task finishes execution
699 //
700 // gtid: global thread ID for calling thread
701 // task: task to be finished
702 // resumed_task: task to be resumed. (may be NULL if task is serialized)
703 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
704  kmp_taskdata_t *resumed_task) {
705  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
706  kmp_info_t *thread = __kmp_threads[gtid];
707  kmp_task_team_t *task_team =
708  thread->th.th_task_team; // might be NULL for serial teams...
709  kmp_int32 children = 0;
710 
711  KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "
712  "task %p\n",
713  gtid, taskdata, resumed_task));
714 
715  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
716 
717 // Pop task from stack if tied
718 #ifdef BUILD_TIED_TASK_STACK
719  if (taskdata->td_flags.tiedness == TASK_TIED) {
720  __kmp_pop_task_stack(gtid, thread, taskdata);
721  }
722 #endif /* BUILD_TIED_TASK_STACK */
723 
724  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
725  // untied task needs to check the counter so that the task structure is not
726  // freed prematurely
727  kmp_int32 counter = KMP_TEST_THEN_DEC32(&taskdata->td_untied_count) - 1;
728  KA_TRACE(
729  20,
730  ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n",
731  gtid, counter, taskdata));
732  if (counter > 0) {
733  // untied task is not done, to be continued possibly by other thread, do
734  // not free it now
735  if (resumed_task == NULL) {
736  KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial);
737  resumed_task = taskdata->td_parent; // In a serialized task, the resumed
738  // task is the parent
739  }
740  thread->th.th_current_task = resumed_task; // restore current_task
741  resumed_task->td_flags.executing = 1; // resume previous task
742  KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, "
743  "resuming task %p\n",
744  gtid, taskdata, resumed_task));
745  return;
746  }
747  }
748 
749  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
750  taskdata->td_flags.complete = 1; // mark the task as completed
751  KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1);
752  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
753 
754  // Only need to keep track of count if team parallel and tasking not
755  // serialized
756  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
757  // Predecrement simulated by "- 1" calculation
758  children =
759  KMP_TEST_THEN_DEC32(&taskdata->td_parent->td_incomplete_child_tasks) -
760  1;
761  KMP_DEBUG_ASSERT(children >= 0);
762 #if OMP_40_ENABLED
763  if (taskdata->td_taskgroup)
764  KMP_TEST_THEN_DEC32((kmp_int32 *)(&taskdata->td_taskgroup->count));
765 #if OMP_45_ENABLED
766  }
767  // if we found proxy tasks there could exist a dependency chain
768  // with the proxy task as origin
769  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) ||
770  (task_team && task_team->tt.tt_found_proxy_tasks)) {
771 #endif
772  __kmp_release_deps(gtid, taskdata);
773 #endif
774  }
775 
776  // td_flags.executing must be marked as 0 after __kmp_release_deps has been
777  // called. Othertwise, if a task is executed immediately from the release_deps
778  // code, the flag will be reset to 1 again by this same function
779  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1);
780  taskdata->td_flags.executing = 0; // suspend the finishing task
781 
782  KA_TRACE(
783  20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
784  gtid, taskdata, children));
785 
786 #if OMP_40_ENABLED
787  /* If the tasks' destructor thunk flag has been set, we need to invoke the
788  destructor thunk that has been generated by the compiler. The code is
789  placed here, since at this point other tasks might have been released
790  hence overlapping the destructor invokations with some other work in the
791  released tasks. The OpenMP spec is not specific on when the destructors
792  are invoked, so we should be free to choose. */
793  if (taskdata->td_flags.destructors_thunk) {
794  kmp_routine_entry_t destr_thunk = task->data1.destructors;
795  KMP_ASSERT(destr_thunk);
796  destr_thunk(gtid, task);
797  }
798 #endif // OMP_40_ENABLED
799 
800  // bookkeeping for resuming task:
801  // GEH - note tasking_ser => task_serial
802  KMP_DEBUG_ASSERT(
803  (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==
804  taskdata->td_flags.task_serial);
805  if (taskdata->td_flags.task_serial) {
806  if (resumed_task == NULL) {
807  resumed_task = taskdata->td_parent; // In a serialized task, the resumed
808  // task is the parent
809  }
810  } else {
811  KMP_DEBUG_ASSERT(resumed_task !=
812  NULL); // verify that resumed task is passed as arguemnt
813  }
814 
815  // Free this task and then ancestor tasks if they have no children.
816  // Restore th_current_task first as suggested by John:
817  // johnmc: if an asynchronous inquiry peers into the runtime system
818  // it doesn't see the freed task as the current task.
819  thread->th.th_current_task = resumed_task;
820  __kmp_free_task_and_ancestors(gtid, taskdata, thread);
821 
822  // TODO: GEH - make sure root team implicit task is initialized properly.
823  // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 );
824  resumed_task->td_flags.executing = 1; // resume previous task
825 
826  KA_TRACE(
827  10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
828  gtid, taskdata, resumed_task));
829 
830  return;
831 }
832 
833 template <bool ompt>
834 static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref,
835  kmp_int32 gtid,
836  kmp_task_t *task) {
837  KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
838  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
839  // this routine will provide task to resume
840  __kmp_task_finish(gtid, task, NULL);
841 
842  KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
843  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
844 
845 #if OMPT_SUPPORT
846  if (ompt) {
847  __ompt_task_finish(task, NULL);
848  omp_frame_t *ompt_frame;
849  __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
850  ompt_frame->enter_frame = NULL;
851  }
852 #endif
853 
854  return;
855 }
856 
857 #if OMPT_SUPPORT
858 OMPT_NOINLINE
859 void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
860  kmp_task_t *task) {
861  __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task);
862 }
863 #endif // OMPT_SUPPORT
864 
865 // __kmpc_omp_task_complete_if0: report that a task has completed execution
866 //
867 // loc_ref: source location information; points to end of task block.
868 // gtid: global thread number.
869 // task: task thunk for the completed task.
870 void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid,
871  kmp_task_t *task) {
872 #if OMPT_SUPPORT
873  if (UNLIKELY(ompt_enabled.enabled)) {
874  __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task);
875  return;
876  }
877 #endif
878  __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task);
879 }
880 
881 #ifdef TASK_UNUSED
882 // __kmpc_omp_task_complete: report that a task has completed execution
883 // NEVER GENERATED BY COMPILER, DEPRECATED!!!
884 void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid,
885  kmp_task_t *task) {
886  KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid,
887  loc_ref, KMP_TASK_TO_TASKDATA(task)));
888 
889  __kmp_task_finish(gtid, task, NULL); // Not sure how to find task to resume
890 
891  KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid,
892  loc_ref, KMP_TASK_TO_TASKDATA(task)));
893  return;
894 }
895 #endif // TASK_UNUSED
896 
897 // __kmp_init_implicit_task: Initialize the appropriate fields in the implicit
898 // task for a given thread
899 //
900 // loc_ref: reference to source location of parallel region
901 // this_thr: thread data structure corresponding to implicit task
902 // team: team for this_thr
903 // tid: thread id of given thread within team
904 // set_curr_task: TRUE if need to push current task to thread
905 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to
906 // have already been done elsewhere.
907 // TODO: Get better loc_ref. Value passed in may be NULL
908 void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
909  kmp_team_t *team, int tid, int set_curr_task) {
910  kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid];
911 
912  KF_TRACE(
913  10,
914  ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
915  tid, team, task, set_curr_task ? "TRUE" : "FALSE"));
916 
917  task->td_task_id = KMP_GEN_TASK_ID();
918  task->td_team = team;
919  // task->td_parent = NULL; // fix for CQ230101 (broken parent task info
920  // in debugger)
921  task->td_ident = loc_ref;
922  task->td_taskwait_ident = NULL;
923  task->td_taskwait_counter = 0;
924  task->td_taskwait_thread = 0;
925 
926  task->td_flags.tiedness = TASK_TIED;
927  task->td_flags.tasktype = TASK_IMPLICIT;
928 #if OMP_45_ENABLED
929  task->td_flags.proxy = TASK_FULL;
930 #endif
931 
932  // All implicit tasks are executed immediately, not deferred
933  task->td_flags.task_serial = 1;
934  task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
935  task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
936 
937  task->td_flags.started = 1;
938  task->td_flags.executing = 1;
939  task->td_flags.complete = 0;
940  task->td_flags.freed = 0;
941 
942 #if OMP_40_ENABLED
943  task->td_depnode = NULL;
944 #endif
945  task->td_last_tied = task;
946 
947  if (set_curr_task) { // only do this init first time thread is created
948  task->td_incomplete_child_tasks = 0;
949  // Not used: don't need to deallocate implicit task
950  task->td_allocated_child_tasks = 0;
951 #if OMP_40_ENABLED
952  task->td_taskgroup = NULL; // An implicit task does not have taskgroup
953  task->td_dephash = NULL;
954 #endif
955  __kmp_push_current_task_to_thread(this_thr, team, tid);
956  } else {
957  KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
958  KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
959  }
960 
961 #if OMPT_SUPPORT
962  if (UNLIKELY(ompt_enabled.enabled))
963  __ompt_task_init(task, tid);
964 #endif
965 
966  KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid,
967  team, task));
968 }
969 
970 // __kmp_finish_implicit_task: Release resources associated to implicit tasks
971 // at the end of parallel regions. Some resources are kept for reuse in the next
972 // parallel region.
973 //
974 // thread: thread data structure corresponding to implicit task
975 void __kmp_finish_implicit_task(kmp_info_t *thread) {
976  kmp_taskdata_t *task = thread->th.th_current_task;
977  if (task->td_dephash)
978  __kmp_dephash_free_entries(thread, task->td_dephash);
979 }
980 
981 // __kmp_free_implicit_task: Release resources associated to implicit tasks
982 // when these are destroyed regions
983 //
984 // thread: thread data structure corresponding to implicit task
985 void __kmp_free_implicit_task(kmp_info_t *thread) {
986  kmp_taskdata_t *task = thread->th.th_current_task;
987  if (task && task->td_dephash) {
988  __kmp_dephash_free(thread, task->td_dephash);
989  task->td_dephash = NULL;
990  }
991 }
992 
993 // Round up a size to a power of two specified by val: Used to insert padding
994 // between structures co-allocated using a single malloc() call
995 static size_t __kmp_round_up_to_val(size_t size, size_t val) {
996  if (size & (val - 1)) {
997  size &= ~(val - 1);
998  if (size <= KMP_SIZE_T_MAX - val) {
999  size += val; // Round up if there is no overflow.
1000  }
1001  }
1002  return size;
1003 } // __kmp_round_up_to_va
1004 
1005 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task
1006 //
1007 // loc_ref: source location information
1008 // gtid: global thread number.
1009 // flags: include tiedness & task type (explicit vs. implicit) of the ''new''
1010 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine.
1011 // sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including
1012 // private vars accessed in task.
1013 // sizeof_shareds: Size in bytes of array of pointers to shared vars accessed
1014 // in task.
1015 // task_entry: Pointer to task code entry point generated by compiler.
1016 // returns: a pointer to the allocated kmp_task_t structure (task).
1017 kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1018  kmp_tasking_flags_t *flags,
1019  size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1020  kmp_routine_entry_t task_entry) {
1021  kmp_task_t *task;
1022  kmp_taskdata_t *taskdata;
1023  kmp_info_t *thread = __kmp_threads[gtid];
1024  kmp_team_t *team = thread->th.th_team;
1025  kmp_taskdata_t *parent_task = thread->th.th_current_task;
1026  size_t shareds_offset;
1027 
1028  KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
1029  "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1030  gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
1031  sizeof_shareds, task_entry));
1032 
1033  if (parent_task->td_flags.final) {
1034  if (flags->merged_if0) {
1035  }
1036  flags->final = 1;
1037  }
1038  if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) {
1039  // Untied task encountered causes the TSC algorithm to check entire deque of
1040  // the victim thread. If no untied task encountered, then checking the head
1041  // of the deque should be enough.
1042  KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1);
1043  }
1044 
1045 #if OMP_45_ENABLED
1046  if (flags->proxy == TASK_PROXY) {
1047  flags->tiedness = TASK_UNTIED;
1048  flags->merged_if0 = 1;
1049 
1050  /* are we running in a sequential parallel or tskm_immediate_exec... we need
1051  tasking support enabled */
1052  if ((thread->th.th_task_team) == NULL) {
1053  /* This should only happen if the team is serialized
1054  setup a task team and propagate it to the thread */
1055  KMP_DEBUG_ASSERT(team->t.t_serialized);
1056  KA_TRACE(30,
1057  ("T#%d creating task team in __kmp_task_alloc for proxy task\n",
1058  gtid));
1059  __kmp_task_team_setup(
1060  thread, team,
1061  1); // 1 indicates setup the current team regardless of nthreads
1062  thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state];
1063  }
1064  kmp_task_team_t *task_team = thread->th.th_task_team;
1065 
1066  /* tasking must be enabled now as the task might not be pushed */
1067  if (!KMP_TASKING_ENABLED(task_team)) {
1068  KA_TRACE(
1069  30,
1070  ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid));
1071  __kmp_enable_tasking(task_team, thread);
1072  kmp_int32 tid = thread->th.th_info.ds.ds_tid;
1073  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
1074  // No lock needed since only owner can allocate
1075  if (thread_data->td.td_deque == NULL) {
1076  __kmp_alloc_task_deque(thread, thread_data);
1077  }
1078  }
1079 
1080  if (task_team->tt.tt_found_proxy_tasks == FALSE)
1081  TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE);
1082  }
1083 #endif
1084 
1085  // Calculate shared structure offset including padding after kmp_task_t struct
1086  // to align pointers in shared struct
1087  shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
1088  shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *));
1089 
1090  // Allocate a kmp_taskdata_t block and a kmp_task_t block.
1091  KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid,
1092  shareds_offset));
1093  KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid,
1094  sizeof_shareds));
1095 
1096 // Avoid double allocation here by combining shareds with taskdata
1097 #if USE_FAST_MEMORY
1098  taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset +
1099  sizeof_shareds);
1100 #else /* ! USE_FAST_MEMORY */
1101  taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset +
1102  sizeof_shareds);
1103 #endif /* USE_FAST_MEMORY */
1104  ANNOTATE_HAPPENS_AFTER(taskdata);
1105 
1106  task = KMP_TASKDATA_TO_TASK(taskdata);
1107 
1108 // Make sure task & taskdata are aligned appropriately
1109 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD
1110  KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0);
1111  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0);
1112 #else
1113  KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0);
1114  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0);
1115 #endif
1116  if (sizeof_shareds > 0) {
1117  // Avoid double allocation here by combining shareds with taskdata
1118  task->shareds = &((char *)taskdata)[shareds_offset];
1119  // Make sure shareds struct is aligned to pointer size
1120  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
1121  0);
1122  } else {
1123  task->shareds = NULL;
1124  }
1125  task->routine = task_entry;
1126  task->part_id = 0; // AC: Always start with 0 part id
1127 
1128  taskdata->td_task_id = KMP_GEN_TASK_ID();
1129  taskdata->td_team = team;
1130  taskdata->td_alloc_thread = thread;
1131  taskdata->td_parent = parent_task;
1132  taskdata->td_level = parent_task->td_level + 1; // increment nesting level
1133  taskdata->td_untied_count = 0;
1134  taskdata->td_ident = loc_ref;
1135  taskdata->td_taskwait_ident = NULL;
1136  taskdata->td_taskwait_counter = 0;
1137  taskdata->td_taskwait_thread = 0;
1138  KMP_DEBUG_ASSERT(taskdata->td_parent != NULL);
1139 #if OMP_45_ENABLED
1140  // avoid copying icvs for proxy tasks
1141  if (flags->proxy == TASK_FULL)
1142 #endif
1143  copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs);
1144 
1145  taskdata->td_flags.tiedness = flags->tiedness;
1146  taskdata->td_flags.final = flags->final;
1147  taskdata->td_flags.merged_if0 = flags->merged_if0;
1148 #if OMP_40_ENABLED
1149  taskdata->td_flags.destructors_thunk = flags->destructors_thunk;
1150 #endif // OMP_40_ENABLED
1151 #if OMP_45_ENABLED
1152  taskdata->td_flags.proxy = flags->proxy;
1153  taskdata->td_task_team = thread->th.th_task_team;
1154  taskdata->td_size_alloc = shareds_offset + sizeof_shareds;
1155 #endif
1156  taskdata->td_flags.tasktype = TASK_EXPLICIT;
1157 
1158  // GEH - TODO: fix this to copy parent task's value of tasking_ser flag
1159  taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1160 
1161  // GEH - TODO: fix this to copy parent task's value of team_serial flag
1162  taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1163 
1164  // GEH - Note we serialize the task if the team is serialized to make sure
1165  // implicit parallel region tasks are not left until program termination to
1166  // execute. Also, it helps locality to execute immediately.
1167 
1168  taskdata->td_flags.task_serial =
1169  (parent_task->td_flags.final || taskdata->td_flags.team_serial ||
1170  taskdata->td_flags.tasking_ser);
1171 
1172  taskdata->td_flags.started = 0;
1173  taskdata->td_flags.executing = 0;
1174  taskdata->td_flags.complete = 0;
1175  taskdata->td_flags.freed = 0;
1176 
1177  taskdata->td_flags.native = flags->native;
1178 
1179  taskdata->td_incomplete_child_tasks = 0;
1180  taskdata->td_allocated_child_tasks = 1; // start at one because counts current
1181 // task and children
1182 #if OMP_40_ENABLED
1183  taskdata->td_taskgroup =
1184  parent_task->td_taskgroup; // task inherits taskgroup from the parent task
1185  taskdata->td_dephash = NULL;
1186  taskdata->td_depnode = NULL;
1187 #endif
1188  if (flags->tiedness == TASK_UNTIED)
1189  taskdata->td_last_tied = NULL; // will be set when the task is scheduled
1190  else
1191  taskdata->td_last_tied = taskdata;
1192 
1193 // Only need to keep track of child task counts if team parallel and tasking not
1194 // serialized or if it is a proxy task
1195 #if OMP_45_ENABLED
1196  if (flags->proxy == TASK_PROXY ||
1197  !(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser))
1198 #else
1199  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser))
1200 #endif
1201  {
1202  KMP_TEST_THEN_INC32(&parent_task->td_incomplete_child_tasks);
1203 #if OMP_40_ENABLED
1204  if (parent_task->td_taskgroup)
1205  KMP_TEST_THEN_INC32((kmp_int32 *)(&parent_task->td_taskgroup->count));
1206 #endif
1207  // Only need to keep track of allocated child tasks for explicit tasks since
1208  // implicit not deallocated
1209  if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) {
1210  KMP_TEST_THEN_INC32(&taskdata->td_parent->td_allocated_child_tasks);
1211  }
1212  }
1213 
1214  KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
1215  gtid, taskdata, taskdata->td_parent));
1216  ANNOTATE_HAPPENS_BEFORE(task);
1217 
1218 #if OMPT_SUPPORT
1219  if (UNLIKELY(ompt_enabled.enabled))
1220  __ompt_task_init(taskdata, gtid);
1221 #endif
1222 
1223  return task;
1224 }
1225 
1226 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1227  kmp_int32 flags, size_t sizeof_kmp_task_t,
1228  size_t sizeof_shareds,
1229  kmp_routine_entry_t task_entry) {
1230  kmp_task_t *retval;
1231  kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
1232 
1233  input_flags->native = FALSE;
1234 // __kmp_task_alloc() sets up all other runtime flags
1235 
1236 #if OMP_45_ENABLED
1237  KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s) "
1238  "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1239  gtid, loc_ref, input_flags->tiedness ? "tied " : "untied",
1240  input_flags->proxy ? "proxy" : "", sizeof_kmp_task_t,
1241  sizeof_shareds, task_entry));
1242 #else
1243  KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s) "
1244  "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1245  gtid, loc_ref, input_flags->tiedness ? "tied " : "untied",
1246  sizeof_kmp_task_t, sizeof_shareds, task_entry));
1247 #endif
1248 
1249  retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t,
1250  sizeof_shareds, task_entry);
1251 
1252  KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval));
1253 
1254  return retval;
1255 }
1256 
1257 // __kmp_invoke_task: invoke the specified task
1258 //
1259 // gtid: global thread ID of caller
1260 // task: the task to invoke
1261 // current_task: the task to resume after task invokation
1262 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
1263  kmp_taskdata_t *current_task) {
1264  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
1265  kmp_uint64 cur_time;
1266 #if OMP_40_ENABLED
1267  int discard = 0 /* false */;
1268 #endif
1269  KA_TRACE(
1270  30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
1271  gtid, taskdata, current_task));
1272  KMP_DEBUG_ASSERT(task);
1273 #if OMP_45_ENABLED
1274  if (taskdata->td_flags.proxy == TASK_PROXY &&
1275  taskdata->td_flags.complete == 1) {
1276  // This is a proxy task that was already completed but it needs to run
1277  // its bottom-half finish
1278  KA_TRACE(
1279  30,
1280  ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n",
1281  gtid, taskdata));
1282 
1283  __kmp_bottom_half_finish_proxy(gtid, task);
1284 
1285  KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for "
1286  "proxy task %p, resuming task %p\n",
1287  gtid, taskdata, current_task));
1288 
1289  return;
1290  }
1291 #endif
1292 
1293 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1294  if (__kmp_forkjoin_frames_mode == 3) {
1295  // Get the current time stamp to measure task execution time to correct
1296  // barrier imbalance time
1297  cur_time = __itt_get_timestamp();
1298  }
1299 #endif
1300 
1301 #if OMP_45_ENABLED
1302  // Proxy tasks are not handled by the runtime
1303  if (taskdata->td_flags.proxy != TASK_PROXY) {
1304 #endif
1305  ANNOTATE_HAPPENS_AFTER(task);
1306  __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded
1307 #if OMP_45_ENABLED
1308  }
1309 #endif
1310 
1311 #if OMPT_SUPPORT
1312  ompt_thread_info_t oldInfo;
1313  kmp_info_t *thread;
1314  if (UNLIKELY(ompt_enabled.enabled)) {
1315  // Store the threads states and restore them after the task
1316  thread = __kmp_threads[gtid];
1317  oldInfo = thread->th.ompt_thread_info;
1318  thread->th.ompt_thread_info.wait_id = 0;
1319  thread->th.ompt_thread_info.state = (thread->th.th_team_serialized)
1320  ? omp_state_work_serial
1321  : omp_state_work_parallel;
1322  taskdata->ompt_task_info.frame.exit_frame = OMPT_GET_FRAME_ADDRESS(0);
1323  }
1324 #endif
1325 
1326 #if OMP_40_ENABLED
1327  // TODO: cancel tasks if the parallel region has also been cancelled
1328  // TODO: check if this sequence can be hoisted above __kmp_task_start
1329  // if cancellation has been enabled for this run ...
1330  if (__kmp_omp_cancellation) {
1331  kmp_info_t *this_thr = __kmp_threads[gtid];
1332  kmp_team_t *this_team = this_thr->th.th_team;
1333  kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
1334  if ((taskgroup && taskgroup->cancel_request) ||
1335  (this_team->t.t_cancel_request == cancel_parallel)) {
1336 #if OMPT_SUPPORT && OMPT_OPTIONAL
1337  ompt_data_t *task_data;
1338  if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) {
1339  __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
1340  ompt_callbacks.ompt_callback(ompt_callback_cancel)(
1341  task_data,
1342  ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup
1343  : ompt_cancel_parallel) |
1344  ompt_cancel_discarded_task,
1345  NULL);
1346  }
1347 #endif
1348  KMP_COUNT_BLOCK(TASK_cancelled);
1349  // this task belongs to a task group and we need to cancel it
1350  discard = 1 /* true */;
1351  }
1352  }
1353 
1354  // Invoke the task routine and pass in relevant data.
1355  // Thunks generated by gcc take a different argument list.
1356  if (!discard) {
1357  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
1358  taskdata->td_last_tied = current_task->td_last_tied;
1359  KMP_DEBUG_ASSERT(taskdata->td_last_tied);
1360  }
1361 #if KMP_STATS_ENABLED
1362  KMP_COUNT_BLOCK(TASK_executed);
1363  switch (KMP_GET_THREAD_STATE()) {
1364  case FORK_JOIN_BARRIER:
1365  KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar);
1366  break;
1367  case PLAIN_BARRIER:
1368  KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar);
1369  break;
1370  case TASKYIELD:
1371  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield);
1372  break;
1373  case TASKWAIT:
1374  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait);
1375  break;
1376  case TASKGROUP:
1377  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup);
1378  break;
1379  default:
1380  KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate);
1381  break;
1382  }
1383 #endif // KMP_STATS_ENABLED
1384 #endif // OMP_40_ENABLED
1385 
1386 // OMPT task begin
1387 #if OMPT_SUPPORT
1388  if (UNLIKELY(ompt_enabled.enabled))
1389  __ompt_task_start(task, current_task, gtid);
1390 #endif
1391 
1392 #ifdef KMP_GOMP_COMPAT
1393  if (taskdata->td_flags.native) {
1394  ((void (*)(void *))(*(task->routine)))(task->shareds);
1395  } else
1396 #endif /* KMP_GOMP_COMPAT */
1397  {
1398  (*(task->routine))(gtid, task);
1399  }
1400  KMP_POP_PARTITIONED_TIMER();
1401 
1402 #if OMPT_SUPPORT
1403  if (UNLIKELY(ompt_enabled.enabled))
1404  __ompt_task_finish(task, current_task);
1405 #endif
1406 #if OMP_40_ENABLED
1407  }
1408 #endif // OMP_40_ENABLED
1409 
1410 #if OMPT_SUPPORT
1411  if (UNLIKELY(ompt_enabled.enabled)) {
1412  thread->th.ompt_thread_info = oldInfo;
1413  taskdata->ompt_task_info.frame.exit_frame = NULL;
1414  }
1415 #endif
1416 
1417 #if OMP_45_ENABLED
1418  // Proxy tasks are not handled by the runtime
1419  if (taskdata->td_flags.proxy != TASK_PROXY) {
1420 #endif
1421  ANNOTATE_HAPPENS_BEFORE(taskdata->td_parent);
1422  __kmp_task_finish(gtid, task, current_task); // OMPT only if not discarded
1423 #if OMP_45_ENABLED
1424  }
1425 #endif
1426 
1427 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1428  // Barrier imbalance - correct arrive time after the task finished
1429  if (__kmp_forkjoin_frames_mode == 3) {
1430  kmp_info_t *this_thr = __kmp_threads[gtid];
1431  if (this_thr->th.th_bar_arrive_time) {
1432  this_thr->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time);
1433  }
1434  }
1435 #endif
1436  KA_TRACE(
1437  30,
1438  ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n",
1439  gtid, taskdata, current_task));
1440  return;
1441 }
1442 
1443 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution
1444 //
1445 // loc_ref: location of original task pragma (ignored)
1446 // gtid: Global Thread ID of encountering thread
1447 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task''
1448 // Returns:
1449 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1450 // be resumed later.
1451 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1452 // resumed later.
1453 kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid,
1454  kmp_task_t *new_task) {
1455  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1456 
1457  KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid,
1458  loc_ref, new_taskdata));
1459 
1460 #if OMPT_SUPPORT
1461  kmp_taskdata_t *parent;
1462  if (UNLIKELY(ompt_enabled.enabled)) {
1463  parent = new_taskdata->td_parent;
1464  if (ompt_enabled.ompt_callback_task_create) {
1465  ompt_data_t task_data = ompt_data_none;
1466  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1467  parent ? &(parent->ompt_task_info.task_data) : &task_data,
1468  parent ? &(parent->ompt_task_info.frame) : NULL,
1469  &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0,
1470  OMPT_GET_RETURN_ADDRESS(0));
1471  }
1472  }
1473 #endif
1474 
1475  /* Should we execute the new task or queue it? For now, let's just always try
1476  to queue it. If the queue fills up, then we'll execute it. */
1477 
1478  if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
1479  { // Execute this task immediately
1480  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1481  new_taskdata->td_flags.task_serial = 1;
1482  __kmp_invoke_task(gtid, new_task, current_task);
1483  }
1484 
1485  KA_TRACE(
1486  10,
1487  ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
1488  "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n",
1489  gtid, loc_ref, new_taskdata));
1490 
1491  ANNOTATE_HAPPENS_BEFORE(new_task);
1492 #if OMPT_SUPPORT
1493  if (UNLIKELY(ompt_enabled.enabled)) {
1494  parent->ompt_task_info.frame.enter_frame = NULL;
1495  }
1496 #endif
1497  return TASK_CURRENT_NOT_QUEUED;
1498 }
1499 
1500 // __kmp_omp_task: Schedule a non-thread-switchable task for execution
1501 //
1502 // gtid: Global Thread ID of encountering thread
1503 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc()
1504 // serialize_immediate: if TRUE then if the task is executed immediately its
1505 // execution will be serialized
1506 // Returns:
1507 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1508 // be resumed later.
1509 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1510 // resumed later.
1511 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
1512  bool serialize_immediate) {
1513  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1514 
1515 /* Should we execute the new task or queue it? For now, let's just always try to
1516  queue it. If the queue fills up, then we'll execute it. */
1517 #if OMP_45_ENABLED
1518  if (new_taskdata->td_flags.proxy == TASK_PROXY ||
1519  __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
1520 #else
1521  if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
1522 #endif
1523  { // Execute this task immediately
1524  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1525  if (serialize_immediate)
1526  new_taskdata->td_flags.task_serial = 1;
1527  __kmp_invoke_task(gtid, new_task, current_task);
1528  }
1529 
1530  ANNOTATE_HAPPENS_BEFORE(new_task);
1531  return TASK_CURRENT_NOT_QUEUED;
1532 }
1533 
1534 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a
1535 // non-thread-switchable task from the parent thread only!
1536 //
1537 // loc_ref: location of original task pragma (ignored)
1538 // gtid: Global Thread ID of encountering thread
1539 // new_task: non-thread-switchable task thunk allocated by
1540 // __kmp_omp_task_alloc()
1541 // Returns:
1542 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1543 // be resumed later.
1544 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1545 // resumed later.
1546 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
1547  kmp_task_t *new_task) {
1548  kmp_int32 res;
1549  KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK);
1550 
1551 #if KMP_DEBUG || OMPT_SUPPORT
1552  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1553 #endif
1554  KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
1555  new_taskdata));
1556 
1557 #if OMPT_SUPPORT
1558  kmp_taskdata_t *parent = NULL;
1559  if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) {
1560  OMPT_STORE_RETURN_ADDRESS(gtid);
1561  parent = new_taskdata->td_parent;
1562  if (!parent->ompt_task_info.frame.enter_frame)
1563  parent->ompt_task_info.frame.enter_frame = OMPT_GET_FRAME_ADDRESS(1);
1564  if (ompt_enabled.ompt_callback_task_create) {
1565  ompt_data_t task_data = ompt_data_none;
1566  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1567  parent ? &(parent->ompt_task_info.task_data) : &task_data,
1568  parent ? &(parent->ompt_task_info.frame) : NULL,
1569  &(new_taskdata->ompt_task_info.task_data),
1570  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
1571  OMPT_LOAD_RETURN_ADDRESS(gtid));
1572  }
1573  }
1574 #endif
1575 
1576  res = __kmp_omp_task(gtid, new_task, true);
1577 
1578  KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning "
1579  "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1580  gtid, loc_ref, new_taskdata));
1581 #if OMPT_SUPPORT
1582  if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
1583  parent->ompt_task_info.frame.enter_frame = NULL;
1584  }
1585 #endif
1586  return res;
1587 }
1588 
1589 template <bool ompt>
1590 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
1591  void *frame_address,
1592  void *return_address) {
1593  kmp_taskdata_t *taskdata;
1594  kmp_info_t *thread;
1595  int thread_finished = FALSE;
1596  KMP_SET_THREAD_STATE_BLOCK(TASKWAIT);
1597 
1598  KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref));
1599 
1600  if (__kmp_tasking_mode != tskm_immediate_exec) {
1601  thread = __kmp_threads[gtid];
1602  taskdata = thread->th.th_current_task;
1603 
1604 #if OMPT_SUPPORT && OMPT_OPTIONAL
1605  ompt_data_t *my_task_data;
1606  ompt_data_t *my_parallel_data;
1607 
1608  if (ompt) {
1609  my_task_data = &(taskdata->ompt_task_info.task_data);
1610  my_parallel_data = OMPT_CUR_TEAM_DATA(thread);
1611 
1612  taskdata->ompt_task_info.frame.enter_frame = frame_address;
1613 
1614  if (ompt_enabled.ompt_callback_sync_region) {
1615  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
1616  ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
1617  my_task_data, return_address);
1618  }
1619 
1620  if (ompt_enabled.ompt_callback_sync_region_wait) {
1621  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
1622  ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
1623  my_task_data, return_address);
1624  }
1625  }
1626 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
1627 
1628 // Debugger: The taskwait is active. Store location and thread encountered the
1629 // taskwait.
1630 #if USE_ITT_BUILD
1631 // Note: These values are used by ITT events as well.
1632 #endif /* USE_ITT_BUILD */
1633  taskdata->td_taskwait_counter += 1;
1634  taskdata->td_taskwait_ident = loc_ref;
1635  taskdata->td_taskwait_thread = gtid + 1;
1636 
1637 #if USE_ITT_BUILD
1638  void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
1639  if (itt_sync_obj != NULL)
1640  __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
1641 #endif /* USE_ITT_BUILD */
1642 
1643  bool must_wait =
1644  !taskdata->td_flags.team_serial && !taskdata->td_flags.final;
1645 
1646 #if OMP_45_ENABLED
1647  must_wait = must_wait || (thread->th.th_task_team != NULL &&
1648  thread->th.th_task_team->tt.tt_found_proxy_tasks);
1649 #endif
1650  if (must_wait) {
1651  kmp_flag_32 flag(
1652  RCAST(volatile kmp_uint32 *, &taskdata->td_incomplete_child_tasks),
1653  0U);
1654  while (TCR_4(taskdata->td_incomplete_child_tasks) != 0) {
1655  flag.execute_tasks(thread, gtid, FALSE,
1656  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
1657  __kmp_task_stealing_constraint);
1658  }
1659  }
1660 #if USE_ITT_BUILD
1661  if (itt_sync_obj != NULL)
1662  __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
1663 #endif /* USE_ITT_BUILD */
1664 
1665  // Debugger: The taskwait is completed. Location remains, but thread is
1666  // negated.
1667  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
1668 
1669 #if OMPT_SUPPORT && OMPT_OPTIONAL
1670  if (ompt) {
1671  if (ompt_enabled.ompt_callback_sync_region_wait) {
1672  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
1673  ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
1674  my_task_data, return_address);
1675  }
1676  if (ompt_enabled.ompt_callback_sync_region) {
1677  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
1678  ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
1679  my_task_data, return_address);
1680  }
1681  taskdata->ompt_task_info.frame.enter_frame = NULL;
1682  }
1683 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
1684 
1685  ANNOTATE_HAPPENS_AFTER(taskdata);
1686  }
1687 
1688  KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
1689  "returning TASK_CURRENT_NOT_QUEUED\n",
1690  gtid, taskdata));
1691 
1692  return TASK_CURRENT_NOT_QUEUED;
1693 }
1694 
1695 #if OMPT_SUPPORT
1696 OMPT_NOINLINE
1697 static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid,
1698  void *frame_address,
1699  void *return_address) {
1700  return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address,
1701  return_address);
1702 }
1703 #endif // OMPT_SUPPORT
1704 
1705 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are
1706 // complete
1707 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) {
1708 #if OMPT_SUPPORT && OMPT_OPTIONAL
1709  if (UNLIKELY(ompt_enabled.enabled)) {
1710  OMPT_STORE_RETURN_ADDRESS(gtid);
1711  return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(1),
1712  OMPT_LOAD_RETURN_ADDRESS(gtid));
1713  }
1714 #endif
1715  return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL);
1716 }
1717 
1718 // __kmpc_omp_taskyield: switch to a different task
1719 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
1720  kmp_taskdata_t *taskdata;
1721  kmp_info_t *thread;
1722  int thread_finished = FALSE;
1723 
1724  KMP_COUNT_BLOCK(OMP_TASKYIELD);
1725  KMP_SET_THREAD_STATE_BLOCK(TASKYIELD);
1726 
1727  KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
1728  gtid, loc_ref, end_part));
1729 
1730  if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) {
1731  thread = __kmp_threads[gtid];
1732  taskdata = thread->th.th_current_task;
1733 // Should we model this as a task wait or not?
1734 // Debugger: The taskwait is active. Store location and thread encountered the
1735 // taskwait.
1736 #if USE_ITT_BUILD
1737 // Note: These values are used by ITT events as well.
1738 #endif /* USE_ITT_BUILD */
1739  taskdata->td_taskwait_counter += 1;
1740  taskdata->td_taskwait_ident = loc_ref;
1741  taskdata->td_taskwait_thread = gtid + 1;
1742 
1743 #if USE_ITT_BUILD
1744  void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
1745  if (itt_sync_obj != NULL)
1746  __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
1747 #endif /* USE_ITT_BUILD */
1748  if (!taskdata->td_flags.team_serial) {
1749  kmp_task_team_t *task_team = thread->th.th_task_team;
1750  if (task_team != NULL) {
1751  if (KMP_TASKING_ENABLED(task_team)) {
1752 #if OMPT_SUPPORT
1753  if (UNLIKELY(ompt_enabled.enabled))
1754  thread->th.ompt_thread_info.ompt_task_yielded = 1;
1755 #endif
1756  __kmp_execute_tasks_32(
1757  thread, gtid, NULL, FALSE,
1758  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
1759  __kmp_task_stealing_constraint);
1760 #if OMPT_SUPPORT
1761  if (UNLIKELY(ompt_enabled.enabled))
1762  thread->th.ompt_thread_info.ompt_task_yielded = 0;
1763 #endif
1764  }
1765  }
1766  }
1767 #if USE_ITT_BUILD
1768  if (itt_sync_obj != NULL)
1769  __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
1770 #endif /* USE_ITT_BUILD */
1771 
1772  // Debugger: The taskwait is completed. Location remains, but thread is
1773  // negated.
1774  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
1775  }
1776 
1777  KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
1778  "returning TASK_CURRENT_NOT_QUEUED\n",
1779  gtid, taskdata));
1780 
1781  return TASK_CURRENT_NOT_QUEUED;
1782 }
1783 
1784 // TODO: change to OMP_50_ENABLED, need to change build tools for this to work
1785 #if OMP_45_ENABLED
1786 // Task Reduction implementation
1787 
1788 typedef struct kmp_task_red_flags {
1789  unsigned lazy_priv : 1; // hint: (1) use lazy allocation (big objects)
1790  unsigned reserved31 : 31;
1791 } kmp_task_red_flags_t;
1792 
1793 // internal structure for reduction data item related info
1794 typedef struct kmp_task_red_data {
1795  void *reduce_shar; // shared reduction item
1796  size_t reduce_size; // size of data item
1797  void *reduce_priv; // thread specific data
1798  void *reduce_pend; // end of private data for comparison op
1799  void *reduce_init; // data initialization routine
1800  void *reduce_fini; // data finalization routine
1801  void *reduce_comb; // data combiner routine
1802  kmp_task_red_flags_t flags; // flags for additional info from compiler
1803 } kmp_task_red_data_t;
1804 
1805 // structure sent us by compiler - one per reduction item
1806 typedef struct kmp_task_red_input {
1807  void *reduce_shar; // shared reduction item
1808  size_t reduce_size; // size of data item
1809  void *reduce_init; // data initialization routine
1810  void *reduce_fini; // data finalization routine
1811  void *reduce_comb; // data combiner routine
1812  kmp_task_red_flags_t flags; // flags for additional info from compiler
1813 } kmp_task_red_input_t;
1814 
1824 void *__kmpc_task_reduction_init(int gtid, int num, void *data) {
1825  kmp_info_t *thread = __kmp_threads[gtid];
1826  kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup;
1827  kmp_int32 nth = thread->th.th_team_nproc;
1828  kmp_task_red_input_t *input = (kmp_task_red_input_t *)data;
1829  kmp_task_red_data_t *arr;
1830 
1831  // check input data just in case
1832  KMP_ASSERT(tg != NULL);
1833  KMP_ASSERT(data != NULL);
1834  KMP_ASSERT(num > 0);
1835  if (nth == 1) {
1836  KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n",
1837  gtid, tg));
1838  return (void *)tg;
1839  }
1840  KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n",
1841  gtid, tg, num));
1842  arr = (kmp_task_red_data_t *)__kmp_thread_malloc(
1843  thread, num * sizeof(kmp_task_red_data_t));
1844  for (int i = 0; i < num; ++i) {
1845  void (*f_init)(void *) = (void (*)(void *))(input[i].reduce_init);
1846  size_t size = input[i].reduce_size - 1;
1847  // round the size up to cache line per thread-specific item
1848  size += CACHE_LINE - size % CACHE_LINE;
1849  KMP_ASSERT(input[i].reduce_comb != NULL); // combiner is mandatory
1850  arr[i].reduce_shar = input[i].reduce_shar;
1851  arr[i].reduce_size = size;
1852  arr[i].reduce_init = input[i].reduce_init;
1853  arr[i].reduce_fini = input[i].reduce_fini;
1854  arr[i].reduce_comb = input[i].reduce_comb;
1855  arr[i].flags = input[i].flags;
1856  if (!input[i].flags.lazy_priv) {
1857  // allocate cache-line aligned block and fill it with zeros
1858  arr[i].reduce_priv = __kmp_allocate(nth * size);
1859  arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size;
1860  if (f_init != NULL) {
1861  // initialize thread-specific items
1862  for (int j = 0; j < nth; ++j) {
1863  f_init((char *)(arr[i].reduce_priv) + j * size);
1864  }
1865  }
1866  } else {
1867  // only allocate space for pointers now,
1868  // objects will be lazily allocated/initialized once requested
1869  arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *));
1870  }
1871  }
1872  tg->reduce_data = (void *)arr;
1873  tg->reduce_num_data = num;
1874  return (void *)tg;
1875 }
1876 
1886 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
1887  kmp_info_t *thread = __kmp_threads[gtid];
1888  kmp_int32 nth = thread->th.th_team_nproc;
1889  if (nth == 1)
1890  return data; // nothing to do
1891 
1892  kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp;
1893  if (tg == NULL)
1894  tg = thread->th.th_current_task->td_taskgroup;
1895  KMP_ASSERT(tg != NULL);
1896  kmp_task_red_data_t *arr = (kmp_task_red_data_t *)(tg->reduce_data);
1897  kmp_int32 num = tg->reduce_num_data;
1898  kmp_int32 tid = thread->th.th_info.ds.ds_tid;
1899 
1900  KMP_ASSERT(data != NULL);
1901  while (tg != NULL) {
1902  for (int i = 0; i < num; ++i) {
1903  if (!arr[i].flags.lazy_priv) {
1904  if (data == arr[i].reduce_shar ||
1905  (data >= arr[i].reduce_priv && data < arr[i].reduce_pend))
1906  return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size;
1907  } else {
1908  // check shared location first
1909  void **p_priv = (void **)(arr[i].reduce_priv);
1910  if (data == arr[i].reduce_shar)
1911  goto found;
1912  // check if we get some thread specific location as parameter
1913  for (int j = 0; j < nth; ++j)
1914  if (data == p_priv[j])
1915  goto found;
1916  continue; // not found, continue search
1917  found:
1918  if (p_priv[tid] == NULL) {
1919  // allocate thread specific object lazily
1920  void (*f_init)(void *) = (void (*)(void *))(arr[i].reduce_init);
1921  p_priv[tid] = __kmp_allocate(arr[i].reduce_size);
1922  if (f_init != NULL) {
1923  f_init(p_priv[tid]);
1924  }
1925  }
1926  return p_priv[tid];
1927  }
1928  }
1929  tg = tg->parent;
1930  arr = (kmp_task_red_data_t *)(tg->reduce_data);
1931  num = tg->reduce_num_data;
1932  }
1933  KMP_ASSERT2(0, "Unknown task reduction item");
1934  return NULL; // ERROR, this line never executed
1935 }
1936 
1937 // Finalize task reduction.
1938 // Called from __kmpc_end_taskgroup()
1939 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) {
1940  kmp_int32 nth = th->th.th_team_nproc;
1941  KMP_DEBUG_ASSERT(nth > 1); // should not be called if nth == 1
1942  kmp_task_red_data_t *arr = (kmp_task_red_data_t *)tg->reduce_data;
1943  kmp_int32 num = tg->reduce_num_data;
1944  for (int i = 0; i < num; ++i) {
1945  void *sh_data = arr[i].reduce_shar;
1946  void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini);
1947  void (*f_comb)(void *, void *) =
1948  (void (*)(void *, void *))(arr[i].reduce_comb);
1949  if (!arr[i].flags.lazy_priv) {
1950  void *pr_data = arr[i].reduce_priv;
1951  size_t size = arr[i].reduce_size;
1952  for (int j = 0; j < nth; ++j) {
1953  void *priv_data = (char *)pr_data + j * size;
1954  f_comb(sh_data, priv_data); // combine results
1955  if (f_fini)
1956  f_fini(priv_data); // finalize if needed
1957  }
1958  } else {
1959  void **pr_data = (void **)(arr[i].reduce_priv);
1960  for (int j = 0; j < nth; ++j) {
1961  if (pr_data[j] != NULL) {
1962  f_comb(sh_data, pr_data[j]); // combine results
1963  if (f_fini)
1964  f_fini(pr_data[j]); // finalize if needed
1965  __kmp_free(pr_data[j]);
1966  }
1967  }
1968  }
1969  __kmp_free(arr[i].reduce_priv);
1970  }
1971  __kmp_thread_free(th, arr);
1972  tg->reduce_data = NULL;
1973  tg->reduce_num_data = 0;
1974 }
1975 #endif
1976 
1977 #if OMP_40_ENABLED
1978 // __kmpc_taskgroup: Start a new taskgroup
1979 void __kmpc_taskgroup(ident_t *loc, int gtid) {
1980  kmp_info_t *thread = __kmp_threads[gtid];
1981  kmp_taskdata_t *taskdata = thread->th.th_current_task;
1982  kmp_taskgroup_t *tg_new =
1983  (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t));
1984  KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new));
1985  tg_new->count = 0;
1986  tg_new->cancel_request = cancel_noreq;
1987  tg_new->parent = taskdata->td_taskgroup;
1988 // TODO: change to OMP_50_ENABLED, need to change build tools for this to work
1989 #if OMP_45_ENABLED
1990  tg_new->reduce_data = NULL;
1991  tg_new->reduce_num_data = 0;
1992 #endif
1993  taskdata->td_taskgroup = tg_new;
1994 
1995 #if OMPT_SUPPORT && OMPT_OPTIONAL
1996  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
1997  void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
1998  if (!codeptr)
1999  codeptr = OMPT_GET_RETURN_ADDRESS(0);
2000  kmp_team_t *team = thread->th.th_team;
2001  ompt_data_t my_task_data = taskdata->ompt_task_info.task_data;
2002  // FIXME: I think this is wrong for lwt!
2003  ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data;
2004 
2005  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2006  ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2007  &(my_task_data), codeptr);
2008  }
2009 #endif
2010 }
2011 
2012 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task
2013 // and its descendants are complete
2014 void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
2015  kmp_info_t *thread = __kmp_threads[gtid];
2016  kmp_taskdata_t *taskdata = thread->th.th_current_task;
2017  kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
2018  int thread_finished = FALSE;
2019 
2020 #if OMPT_SUPPORT && OMPT_OPTIONAL
2021  kmp_team_t *team;
2022  ompt_data_t my_task_data;
2023  ompt_data_t my_parallel_data;
2024  void *codeptr;
2025  if (UNLIKELY(ompt_enabled.enabled)) {
2026  team = thread->th.th_team;
2027  my_task_data = taskdata->ompt_task_info.task_data;
2028  // FIXME: I think this is wrong for lwt!
2029  my_parallel_data = team->t.ompt_team_info.parallel_data;
2030  codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2031  if (!codeptr)
2032  codeptr = OMPT_GET_RETURN_ADDRESS(0);
2033  }
2034 #endif
2035 
2036  KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc));
2037  KMP_DEBUG_ASSERT(taskgroup != NULL);
2038  KMP_SET_THREAD_STATE_BLOCK(TASKGROUP);
2039 
2040  if (__kmp_tasking_mode != tskm_immediate_exec) {
2041  // mark task as waiting not on a barrier
2042  taskdata->td_taskwait_counter += 1;
2043  taskdata->td_taskwait_ident = loc;
2044  taskdata->td_taskwait_thread = gtid + 1;
2045 #if USE_ITT_BUILD
2046  // For ITT the taskgroup wait is similar to taskwait until we need to
2047  // distinguish them
2048  void *itt_sync_obj = __kmp_itt_taskwait_object(gtid);
2049  if (itt_sync_obj != NULL)
2050  __kmp_itt_taskwait_starting(gtid, itt_sync_obj);
2051 #endif /* USE_ITT_BUILD */
2052 
2053 #if OMPT_SUPPORT && OMPT_OPTIONAL
2054  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2055  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2056  ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2057  &(my_task_data), codeptr);
2058  }
2059 #endif
2060 
2061 #if OMP_45_ENABLED
2062  if (!taskdata->td_flags.team_serial ||
2063  (thread->th.th_task_team != NULL &&
2064  thread->th.th_task_team->tt.tt_found_proxy_tasks))
2065 #else
2066  if (!taskdata->td_flags.team_serial)
2067 #endif
2068  {
2069  kmp_flag_32 flag(RCAST(kmp_uint32 *, &taskgroup->count), 0U);
2070  while (TCR_4(taskgroup->count) != 0) {
2071  flag.execute_tasks(thread, gtid, FALSE,
2072  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2073  __kmp_task_stealing_constraint);
2074  }
2075  }
2076  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting
2077 
2078 #if OMPT_SUPPORT && OMPT_OPTIONAL
2079  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2080  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2081  ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2082  &(my_task_data), codeptr);
2083  }
2084 #endif
2085 
2086 #if USE_ITT_BUILD
2087  if (itt_sync_obj != NULL)
2088  __kmp_itt_taskwait_finished(gtid, itt_sync_obj);
2089 #endif /* USE_ITT_BUILD */
2090  }
2091  KMP_DEBUG_ASSERT(taskgroup->count == 0);
2092 
2093 // TODO: change to OMP_50_ENABLED, need to change build tools for this to work
2094 #if OMP_45_ENABLED
2095  if (taskgroup->reduce_data != NULL) // need to reduce?
2096  __kmp_task_reduction_fini(thread, taskgroup);
2097 #endif
2098  // Restore parent taskgroup for the current task
2099  taskdata->td_taskgroup = taskgroup->parent;
2100  __kmp_thread_free(thread, taskgroup);
2101 
2102  KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n",
2103  gtid, taskdata));
2104  ANNOTATE_HAPPENS_AFTER(taskdata);
2105 
2106 #if OMPT_SUPPORT && OMPT_OPTIONAL
2107  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
2108  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2109  ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2110  &(my_task_data), codeptr);
2111  }
2112 #endif
2113 }
2114 #endif
2115 
2116 // __kmp_remove_my_task: remove a task from my own deque
2117 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid,
2118  kmp_task_team_t *task_team,
2119  kmp_int32 is_constrained) {
2120  kmp_task_t *task;
2121  kmp_taskdata_t *taskdata;
2122  kmp_thread_data_t *thread_data;
2123  kmp_uint32 tail;
2124 
2125  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2126  KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data !=
2127  NULL); // Caller should check this condition
2128 
2129  thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)];
2130 
2131  KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",
2132  gtid, thread_data->td.td_deque_ntasks,
2133  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2134 
2135  if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
2136  KA_TRACE(10,
2137  ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
2138  "ntasks=%d head=%u tail=%u\n",
2139  gtid, thread_data->td.td_deque_ntasks,
2140  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2141  return NULL;
2142  }
2143 
2144  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2145 
2146  if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
2147  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2148  KA_TRACE(10,
2149  ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
2150  "ntasks=%d head=%u tail=%u\n",
2151  gtid, thread_data->td.td_deque_ntasks,
2152  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2153  return NULL;
2154  }
2155 
2156  tail = (thread_data->td.td_deque_tail - 1) &
2157  TASK_DEQUE_MASK(thread_data->td); // Wrap index.
2158  taskdata = thread_data->td.td_deque[tail];
2159 
2160  if (is_constrained && (taskdata->td_flags.tiedness == TASK_TIED)) {
2161  // we need to check if the candidate obeys task scheduling constraint (TSC)
2162  // only descendant of all deferred tied tasks can be scheduled, checking
2163  // the last one is enough, as it in turn is the descendant of all others
2164  kmp_taskdata_t *current = thread->th.th_current_task->td_last_tied;
2165  KMP_DEBUG_ASSERT(current != NULL);
2166  // check if last tied task is not suspended on barrier
2167  if (current->td_flags.tasktype == TASK_EXPLICIT ||
2168  current->td_taskwait_thread > 0) { // <= 0 on barrier
2169  kmp_int32 level = current->td_level;
2170  kmp_taskdata_t *parent = taskdata->td_parent;
2171  while (parent != current && parent->td_level > level) {
2172  parent = parent->td_parent; // check generation up to the level of the
2173  // current task
2174  KMP_DEBUG_ASSERT(parent != NULL);
2175  }
2176  if (parent != current) {
2177  // The TSC does not allow to steal victim task
2178  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2179  KA_TRACE(10, ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
2180  "ntasks=%d head=%u tail=%u\n",
2181  gtid, thread_data->td.td_deque_ntasks,
2182  thread_data->td.td_deque_head,
2183  thread_data->td.td_deque_tail));
2184  return NULL;
2185  }
2186  }
2187  }
2188 
2189  thread_data->td.td_deque_tail = tail;
2190  TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1);
2191 
2192  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2193 
2194  KA_TRACE(10, ("__kmp_remove_my_task(exit #2): T#%d task %p removed: "
2195  "ntasks=%d head=%u tail=%u\n",
2196  gtid, taskdata, thread_data->td.td_deque_ntasks,
2197  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2198 
2199  task = KMP_TASKDATA_TO_TASK(taskdata);
2200  return task;
2201 }
2202 
2203 // __kmp_steal_task: remove a task from another thread's deque
2204 // Assume that calling thread has already checked existence of
2205 // task_team thread_data before calling this routine.
2206 static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid,
2207  kmp_task_team_t *task_team,
2208  volatile kmp_int32 *unfinished_threads,
2209  int *thread_finished,
2210  kmp_int32 is_constrained) {
2211  kmp_task_t *task;
2212  kmp_taskdata_t *taskdata;
2213  kmp_taskdata_t *current;
2214  kmp_thread_data_t *victim_td, *threads_data;
2215  kmp_int32 level, target;
2216  kmp_int32 victim_tid;
2217 
2218  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2219 
2220  threads_data = task_team->tt.tt_threads_data;
2221  KMP_DEBUG_ASSERT(threads_data != NULL); // Caller should check this condition
2222 
2223  victim_tid = victim_thr->th.th_info.ds.ds_tid;
2224  victim_td = &threads_data[victim_tid];
2225 
2226  KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
2227  "task_team=%p ntasks=%d head=%u tail=%u\n",
2228  gtid, __kmp_gtid_from_thread(victim_thr), task_team,
2229  victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
2230  victim_td->td.td_deque_tail));
2231 
2232  if (TCR_4(victim_td->td.td_deque_ntasks) == 0) {
2233  KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
2234  "task_team=%p ntasks=%d head=%u tail=%u\n",
2235  gtid, __kmp_gtid_from_thread(victim_thr), task_team,
2236  victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
2237  victim_td->td.td_deque_tail));
2238  return NULL;
2239  }
2240 
2241  __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock);
2242 
2243  int ntasks = TCR_4(victim_td->td.td_deque_ntasks);
2244  // Check again after we acquire the lock
2245  if (ntasks == 0) {
2246  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2247  KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
2248  "task_team=%p ntasks=%d head=%u tail=%u\n",
2249  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
2250  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2251  return NULL;
2252  }
2253 
2254  KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL);
2255 
2256  taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head];
2257  if (is_constrained && (taskdata->td_flags.tiedness == TASK_TIED)) {
2258  // we need to check if the candidate obeys task scheduling constraint (TSC)
2259  // only descendant of all deferred tied tasks can be scheduled, checking
2260  // the last one is enough, as it in turn is the descendant of all others
2261  current = __kmp_threads[gtid]->th.th_current_task->td_last_tied;
2262  KMP_DEBUG_ASSERT(current != NULL);
2263  // check if last tied task is not suspended on barrier
2264  if (current->td_flags.tasktype == TASK_EXPLICIT ||
2265  current->td_taskwait_thread > 0) { // <= 0 on barrier
2266  level = current->td_level;
2267  kmp_taskdata_t *parent = taskdata->td_parent;
2268  while (parent != current && parent->td_level > level) {
2269  parent = parent->td_parent; // check generation up to the level of the
2270  // current task
2271  KMP_DEBUG_ASSERT(parent != NULL);
2272  }
2273  if (parent != current) {
2274  if (!task_team->tt.tt_untied_task_encountered) {
2275  // The TSC does not allow to steal victim task
2276  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2277  KA_TRACE(10,
2278  ("__kmp_steal_task(exit #3): T#%d could not steal from "
2279  "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
2280  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
2281  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2282  return NULL;
2283  }
2284  taskdata = NULL; // will check other tasks in victim's deque
2285  }
2286  }
2287  }
2288  if (taskdata != NULL) {
2289  // Bump head pointer and Wrap.
2290  victim_td->td.td_deque_head =
2291  (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td);
2292  } else {
2293  int i;
2294  // walk through victim's deque trying to steal any task
2295  target = victim_td->td.td_deque_head;
2296  for (i = 1; i < ntasks; ++i) {
2297  target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
2298  taskdata = victim_td->td.td_deque[target];
2299  if (taskdata->td_flags.tiedness == TASK_TIED) {
2300  // check if the candidate obeys the TSC
2301  kmp_taskdata_t *parent = taskdata->td_parent;
2302  // check generation up to the level of the current task
2303  while (parent != current && parent->td_level > level) {
2304  parent = parent->td_parent;
2305  KMP_DEBUG_ASSERT(parent != NULL);
2306  }
2307  if (parent != current) {
2308  // The TSC does not allow to steal the candidate
2309  taskdata = NULL;
2310  continue;
2311  } else {
2312  // found victim tied task
2313  break;
2314  }
2315  } else {
2316  // found victim untied task
2317  break;
2318  }
2319  }
2320  if (taskdata == NULL) {
2321  // No appropriate candidate to steal found
2322  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2323  KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from "
2324  "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
2325  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
2326  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2327  return NULL;
2328  }
2329  int prev = target;
2330  for (i = i + 1; i < ntasks; ++i) {
2331  // shift remaining tasks in the deque left by 1
2332  target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
2333  victim_td->td.td_deque[prev] = victim_td->td.td_deque[target];
2334  prev = target;
2335  }
2336  KMP_DEBUG_ASSERT(victim_td->td.td_deque_tail ==
2337  ((target + 1) & TASK_DEQUE_MASK(victim_td->td)));
2338  victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped))
2339  }
2340  if (*thread_finished) {
2341  // We need to un-mark this victim as a finished victim. This must be done
2342  // before releasing the lock, or else other threads (starting with the
2343  // master victim) might be prematurely released from the barrier!!!
2344  kmp_int32 count;
2345 
2346  count = KMP_TEST_THEN_INC32(unfinished_threads);
2347 
2348  KA_TRACE(
2349  20,
2350  ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
2351  gtid, count + 1, task_team));
2352 
2353  *thread_finished = FALSE;
2354  }
2355  TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1);
2356 
2357  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
2358 
2359  KMP_COUNT_BLOCK(TASK_stolen);
2360  KA_TRACE(10,
2361  ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
2362  "task_team=%p ntasks=%d head=%u tail=%u\n",
2363  gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team,
2364  ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
2365 
2366  task = KMP_TASKDATA_TO_TASK(taskdata);
2367  return task;
2368 }
2369 
2370 // __kmp_execute_tasks_template: Choose and execute tasks until either the
2371 // condition is statisfied (return true) or there are none left (return false).
2372 //
2373 // final_spin is TRUE if this is the spin at the release barrier.
2374 // thread_finished indicates whether the thread is finished executing all
2375 // the tasks it has on its deque, and is at the release barrier.
2376 // spinner is the location on which to spin.
2377 // spinner == NULL means only execute a single task and return.
2378 // checker is the value to check to terminate the spin.
2379 template <class C>
2380 static inline int __kmp_execute_tasks_template(
2381  kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin,
2382  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
2383  kmp_int32 is_constrained) {
2384  kmp_task_team_t *task_team = thread->th.th_task_team;
2385  kmp_thread_data_t *threads_data;
2386  kmp_task_t *task;
2387  kmp_info_t *other_thread;
2388  kmp_taskdata_t *current_task = thread->th.th_current_task;
2389  volatile kmp_int32 *unfinished_threads;
2390  kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0,
2391  tid = thread->th.th_info.ds.ds_tid;
2392 
2393  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2394  KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]);
2395 
2396  if (task_team == NULL)
2397  return FALSE;
2398 
2399  KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d "
2400  "*thread_finished=%d\n",
2401  gtid, final_spin, *thread_finished));
2402 
2403  thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
2404  threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
2405  KMP_DEBUG_ASSERT(threads_data != NULL);
2406 
2407  nthreads = task_team->tt.tt_nproc;
2408  unfinished_threads = &(task_team->tt.tt_unfinished_threads);
2409 #if OMP_45_ENABLED
2410  KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks);
2411 #else
2412  KMP_DEBUG_ASSERT(nthreads > 1);
2413 #endif
2414  KMP_DEBUG_ASSERT(TCR_4(*unfinished_threads) >= 0);
2415 
2416  while (1) { // Outer loop keeps trying to find tasks in case of single thread
2417  // getting tasks from target constructs
2418  while (1) { // Inner loop to find a task and execute it
2419  task = NULL;
2420  if (use_own_tasks) { // check on own queue first
2421  task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained);
2422  }
2423  if ((task == NULL) && (nthreads > 1)) { // Steal a task
2424  int asleep = 1;
2425  use_own_tasks = 0;
2426  // Try to steal from the last place I stole from successfully.
2427  if (victim_tid == -2) { // haven't stolen anything yet
2428  victim_tid = threads_data[tid].td.td_deque_last_stolen;
2429  if (victim_tid !=
2430  -1) // if we have a last stolen from victim, get the thread
2431  other_thread = threads_data[victim_tid].td.td_thr;
2432  }
2433  if (victim_tid != -1) { // found last victim
2434  asleep = 0;
2435  } else if (!new_victim) { // no recent steals and we haven't already
2436  // used a new victim; select a random thread
2437  do { // Find a different thread to steal work from.
2438  // Pick a random thread. Initial plan was to cycle through all the
2439  // threads, and only return if we tried to steal from every thread,
2440  // and failed. Arch says that's not such a great idea.
2441  victim_tid = __kmp_get_random(thread) % (nthreads - 1);
2442  if (victim_tid >= tid) {
2443  ++victim_tid; // Adjusts random distribution to exclude self
2444  }
2445  // Found a potential victim
2446  other_thread = threads_data[victim_tid].td.td_thr;
2447  // There is a slight chance that __kmp_enable_tasking() did not wake
2448  // up all threads waiting at the barrier. If victim is sleeping,
2449  // then wake it up. Since we were going to pay the cache miss
2450  // penalty for referencing another thread's kmp_info_t struct
2451  // anyway,
2452  // the check shouldn't cost too much performance at this point. In
2453  // extra barrier mode, tasks do not sleep at the separate tasking
2454  // barrier, so this isn't a problem.
2455  asleep = 0;
2456  if ((__kmp_tasking_mode == tskm_task_teams) &&
2457  (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) &&
2458  (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc)) !=
2459  NULL)) {
2460  asleep = 1;
2461  __kmp_null_resume_wrapper(__kmp_gtid_from_thread(other_thread),
2462  other_thread->th.th_sleep_loc);
2463  // A sleeping thread should not have any tasks on it's queue.
2464  // There is a slight possibility that it resumes, steals a task
2465  // from another thread, which spawns more tasks, all in the time
2466  // that it takes this thread to check => don't write an assertion
2467  // that the victim's queue is empty. Try stealing from a
2468  // different thread.
2469  }
2470  } while (asleep);
2471  }
2472 
2473  if (!asleep) {
2474  // We have a victim to try to steal from
2475  task = __kmp_steal_task(other_thread, gtid, task_team,
2476  unfinished_threads, thread_finished,
2477  is_constrained);
2478  }
2479  if (task != NULL) { // set last stolen to victim
2480  if (threads_data[tid].td.td_deque_last_stolen != victim_tid) {
2481  threads_data[tid].td.td_deque_last_stolen = victim_tid;
2482  // The pre-refactored code did not try more than 1 successful new
2483  // vicitm, unless the last one generated more local tasks;
2484  // new_victim keeps track of this
2485  new_victim = 1;
2486  }
2487  } else { // No tasks found; unset last_stolen
2488  KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1);
2489  victim_tid = -2; // no successful victim found
2490  }
2491  }
2492 
2493  if (task == NULL) // break out of tasking loop
2494  break;
2495 
2496 // Found a task; execute it
2497 #if USE_ITT_BUILD && USE_ITT_NOTIFY
2498  if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
2499  if (itt_sync_obj == NULL) { // we are at fork barrier where we could not
2500  // get the object reliably
2501  itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
2502  }
2503  __kmp_itt_task_starting(itt_sync_obj);
2504  }
2505 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
2506  __kmp_invoke_task(gtid, task, current_task);
2507 #if USE_ITT_BUILD
2508  if (itt_sync_obj != NULL)
2509  __kmp_itt_task_finished(itt_sync_obj);
2510 #endif /* USE_ITT_BUILD */
2511  // If this thread is only partway through the barrier and the condition is
2512  // met, then return now, so that the barrier gather/release pattern can
2513  // proceed. If this thread is in the last spin loop in the barrier,
2514  // waiting to be released, we know that the termination condition will not
2515  // be satisified, so don't waste any cycles checking it.
2516  if (flag == NULL || (!final_spin && flag->done_check())) {
2517  KA_TRACE(
2518  15,
2519  ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
2520  gtid));
2521  return TRUE;
2522  }
2523  if (thread->th.th_task_team == NULL) {
2524  break;
2525  }
2526  // Yield before executing next task
2527  KMP_YIELD(__kmp_library == library_throughput);
2528  // If execution of a stolen task results in more tasks being placed on our
2529  // run queue, reset use_own_tasks
2530  if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) {
2531  KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned "
2532  "other tasks, restart\n",
2533  gtid));
2534  use_own_tasks = 1;
2535  new_victim = 0;
2536  }
2537  }
2538 
2539 // The task source has been exhausted. If in final spin loop of barrier, check
2540 // if termination condition is satisfied.
2541 #if OMP_45_ENABLED
2542  // The work queue may be empty but there might be proxy tasks still
2543  // executing
2544  if (final_spin && TCR_4(current_task->td_incomplete_child_tasks) == 0)
2545 #else
2546  if (final_spin)
2547 #endif
2548  {
2549  // First, decrement the #unfinished threads, if that has not already been
2550  // done. This decrement might be to the spin location, and result in the
2551  // termination condition being satisfied.
2552  if (!*thread_finished) {
2553  kmp_int32 count;
2554 
2555  count = KMP_TEST_THEN_DEC32(unfinished_threads) - 1;
2556  KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec "
2557  "unfinished_threads to %d task_team=%p\n",
2558  gtid, count, task_team));
2559  *thread_finished = TRUE;
2560  }
2561 
2562  // It is now unsafe to reference thread->th.th_team !!!
2563  // Decrementing task_team->tt.tt_unfinished_threads can allow the master
2564  // thread to pass through the barrier, where it might reset each thread's
2565  // th.th_team field for the next parallel region. If we can steal more
2566  // work, we know that this has not happened yet.
2567  if (flag != NULL && flag->done_check()) {
2568  KA_TRACE(
2569  15,
2570  ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
2571  gtid));
2572  return TRUE;
2573  }
2574  }
2575 
2576  // If this thread's task team is NULL, master has recognized that there are
2577  // no more tasks; bail out
2578  if (thread->th.th_task_team == NULL) {
2579  KA_TRACE(15,
2580  ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid));
2581  return FALSE;
2582  }
2583 
2584 #if OMP_45_ENABLED
2585  // We could be getting tasks from target constructs; if this is the only
2586  // thread, keep trying to execute tasks from own queue
2587  if (nthreads == 1)
2588  use_own_tasks = 1;
2589  else
2590 #endif
2591  {
2592  KA_TRACE(15,
2593  ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid));
2594  return FALSE;
2595  }
2596  }
2597 }
2598 
2599 int __kmp_execute_tasks_32(
2600  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32 *flag, int final_spin,
2601  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
2602  kmp_int32 is_constrained) {
2603  return __kmp_execute_tasks_template(
2604  thread, gtid, flag, final_spin,
2605  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
2606 }
2607 
2608 int __kmp_execute_tasks_64(
2609  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64 *flag, int final_spin,
2610  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
2611  kmp_int32 is_constrained) {
2612  return __kmp_execute_tasks_template(
2613  thread, gtid, flag, final_spin,
2614  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
2615 }
2616 
2617 int __kmp_execute_tasks_oncore(
2618  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin,
2619  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
2620  kmp_int32 is_constrained) {
2621  return __kmp_execute_tasks_template(
2622  thread, gtid, flag, final_spin,
2623  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
2624 }
2625 
2626 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the
2627 // next barrier so they can assist in executing enqueued tasks.
2628 // First thread in allocates the task team atomically.
2629 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
2630  kmp_info_t *this_thr) {
2631  kmp_thread_data_t *threads_data;
2632  int nthreads, i, is_init_thread;
2633 
2634  KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n",
2635  __kmp_gtid_from_thread(this_thr)));
2636 
2637  KMP_DEBUG_ASSERT(task_team != NULL);
2638  KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL);
2639 
2640  nthreads = task_team->tt.tt_nproc;
2641  KMP_DEBUG_ASSERT(nthreads > 0);
2642  KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc);
2643 
2644  // Allocate or increase the size of threads_data if necessary
2645  is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team);
2646 
2647  if (!is_init_thread) {
2648  // Some other thread already set up the array.
2649  KA_TRACE(
2650  20,
2651  ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
2652  __kmp_gtid_from_thread(this_thr)));
2653  return;
2654  }
2655  threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
2656  KMP_DEBUG_ASSERT(threads_data != NULL);
2657 
2658  if ((__kmp_tasking_mode == tskm_task_teams) &&
2659  (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) {
2660  // Release any threads sleeping at the barrier, so that they can steal
2661  // tasks and execute them. In extra barrier mode, tasks do not sleep
2662  // at the separate tasking barrier, so this isn't a problem.
2663  for (i = 0; i < nthreads; i++) {
2664  volatile void *sleep_loc;
2665  kmp_info_t *thread = threads_data[i].td.td_thr;
2666 
2667  if (i == this_thr->th.th_info.ds.ds_tid) {
2668  continue;
2669  }
2670  // Since we haven't locked the thread's suspend mutex lock at this
2671  // point, there is a small window where a thread might be putting
2672  // itself to sleep, but hasn't set the th_sleep_loc field yet.
2673  // To work around this, __kmp_execute_tasks_template() periodically checks
2674  // see if other threads are sleeping (using the same random mechanism that
2675  // is used for task stealing) and awakens them if they are.
2676  if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) !=
2677  NULL) {
2678  KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n",
2679  __kmp_gtid_from_thread(this_thr),
2680  __kmp_gtid_from_thread(thread)));
2681  __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc);
2682  } else {
2683  KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",
2684  __kmp_gtid_from_thread(this_thr),
2685  __kmp_gtid_from_thread(thread)));
2686  }
2687  }
2688  }
2689 
2690  KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n",
2691  __kmp_gtid_from_thread(this_thr)));
2692 }
2693 
2694 /* // TODO: Check the comment consistency
2695  * Utility routines for "task teams". A task team (kmp_task_t) is kind of
2696  * like a shadow of the kmp_team_t data struct, with a different lifetime.
2697  * After a child * thread checks into a barrier and calls __kmp_release() from
2698  * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no
2699  * longer assume that the kmp_team_t structure is intact (at any moment, the
2700  * master thread may exit the barrier code and free the team data structure,
2701  * and return the threads to the thread pool).
2702  *
2703  * This does not work with the the tasking code, as the thread is still
2704  * expected to participate in the execution of any tasks that may have been
2705  * spawned my a member of the team, and the thread still needs access to all
2706  * to each thread in the team, so that it can steal work from it.
2707  *
2708  * Enter the existence of the kmp_task_team_t struct. It employs a reference
2709  * counting mechanims, and is allocated by the master thread before calling
2710  * __kmp_<barrier_kind>_release, and then is release by the last thread to
2711  * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes
2712  * of the kmp_task_team_t structs for consecutive barriers can overlap
2713  * (and will, unless the master thread is the last thread to exit the barrier
2714  * release phase, which is not typical).
2715  *
2716  * The existence of such a struct is useful outside the context of tasking,
2717  * but for now, I'm trying to keep it specific to the OMP_30_ENABLED macro,
2718  * so that any performance differences show up when comparing the 2.5 vs. 3.0
2719  * libraries.
2720  *
2721  * We currently use the existence of the threads array as an indicator that
2722  * tasks were spawned since the last barrier. If the structure is to be
2723  * useful outside the context of tasking, then this will have to change, but
2724  * not settting the field minimizes the performance impact of tasking on
2725  * barriers, when no explicit tasks were spawned (pushed, actually).
2726  */
2727 
2728 static kmp_task_team_t *__kmp_free_task_teams =
2729  NULL; // Free list for task_team data structures
2730 // Lock for task team data structures
2731 kmp_bootstrap_lock_t __kmp_task_team_lock =
2732  KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock);
2733 
2734 // __kmp_alloc_task_deque:
2735 // Allocates a task deque for a particular thread, and initialize the necessary
2736 // data structures relating to the deque. This only happens once per thread
2737 // per task team since task teams are recycled. No lock is needed during
2738 // allocation since each thread allocates its own deque.
2739 static void __kmp_alloc_task_deque(kmp_info_t *thread,
2740  kmp_thread_data_t *thread_data) {
2741  __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock);
2742  KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL);
2743 
2744  // Initialize last stolen task field to "none"
2745  thread_data->td.td_deque_last_stolen = -1;
2746 
2747  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0);
2748  KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0);
2749  KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0);
2750 
2751  KE_TRACE(
2752  10,
2753  ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",
2754  __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data));
2755  // Allocate space for task deque, and zero the deque
2756  // Cannot use __kmp_thread_calloc() because threads not around for
2757  // kmp_reap_task_team( ).
2758  thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate(
2759  INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *));
2760  thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE;
2761 }
2762 
2763 // __kmp_realloc_task_deque:
2764 // Re-allocates a task deque for a particular thread, copies the content from
2765 // the old deque and adjusts the necessary data structures relating to the
2766 // deque. This operation must be done with a the deque_lock being held
2767 static void __kmp_realloc_task_deque(kmp_info_t *thread,
2768  kmp_thread_data_t *thread_data) {
2769  kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td);
2770  kmp_int32 new_size = 2 * size;
2771 
2772  KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "
2773  "%d] for thread_data %p\n",
2774  __kmp_gtid_from_thread(thread), size, new_size, thread_data));
2775 
2776  kmp_taskdata_t **new_deque =
2777  (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *));
2778 
2779  int i, j;
2780  for (i = thread_data->td.td_deque_head, j = 0; j < size;
2781  i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++)
2782  new_deque[j] = thread_data->td.td_deque[i];
2783 
2784  __kmp_free(thread_data->td.td_deque);
2785 
2786  thread_data->td.td_deque_head = 0;
2787  thread_data->td.td_deque_tail = size;
2788  thread_data->td.td_deque = new_deque;
2789  thread_data->td.td_deque_size = new_size;
2790 }
2791 
2792 // __kmp_free_task_deque:
2793 // Deallocates a task deque for a particular thread. Happens at library
2794 // deallocation so don't need to reset all thread data fields.
2795 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) {
2796  if (thread_data->td.td_deque != NULL) {
2797  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2798  TCW_4(thread_data->td.td_deque_ntasks, 0);
2799  __kmp_free(thread_data->td.td_deque);
2800  thread_data->td.td_deque = NULL;
2801  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2802  }
2803 
2804 #ifdef BUILD_TIED_TASK_STACK
2805  // GEH: Figure out what to do here for td_susp_tied_tasks
2806  if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) {
2807  __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data);
2808  }
2809 #endif // BUILD_TIED_TASK_STACK
2810 }
2811 
2812 // __kmp_realloc_task_threads_data:
2813 // Allocates a threads_data array for a task team, either by allocating an
2814 // initial array or enlarging an existing array. Only the first thread to get
2815 // the lock allocs or enlarges the array and re-initializes the array eleemnts.
2816 // That thread returns "TRUE", the rest return "FALSE".
2817 // Assumes that the new array size is given by task_team -> tt.tt_nproc.
2818 // The current size is given by task_team -> tt.tt_max_threads.
2819 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
2820  kmp_task_team_t *task_team) {
2821  kmp_thread_data_t **threads_data_p;
2822  kmp_int32 nthreads, maxthreads;
2823  int is_init_thread = FALSE;
2824 
2825  if (TCR_4(task_team->tt.tt_found_tasks)) {
2826  // Already reallocated and initialized.
2827  return FALSE;
2828  }
2829 
2830  threads_data_p = &task_team->tt.tt_threads_data;
2831  nthreads = task_team->tt.tt_nproc;
2832  maxthreads = task_team->tt.tt_max_threads;
2833 
2834  // All threads must lock when they encounter the first task of the implicit
2835  // task region to make sure threads_data fields are (re)initialized before
2836  // used.
2837  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
2838 
2839  if (!TCR_4(task_team->tt.tt_found_tasks)) {
2840  // first thread to enable tasking
2841  kmp_team_t *team = thread->th.th_team;
2842  int i;
2843 
2844  is_init_thread = TRUE;
2845  if (maxthreads < nthreads) {
2846 
2847  if (*threads_data_p != NULL) {
2848  kmp_thread_data_t *old_data = *threads_data_p;
2849  kmp_thread_data_t *new_data = NULL;
2850 
2851  KE_TRACE(
2852  10,
2853  ("__kmp_realloc_task_threads_data: T#%d reallocating "
2854  "threads data for task_team %p, new_size = %d, old_size = %d\n",
2855  __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads));
2856  // Reallocate threads_data to have more elements than current array
2857  // Cannot use __kmp_thread_realloc() because threads not around for
2858  // kmp_reap_task_team( ). Note all new array entries are initialized
2859  // to zero by __kmp_allocate().
2860  new_data = (kmp_thread_data_t *)__kmp_allocate(
2861  nthreads * sizeof(kmp_thread_data_t));
2862  // copy old data to new data
2863  KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t),
2864  (void *)old_data, maxthreads * sizeof(kmp_thread_data_t));
2865 
2866 #ifdef BUILD_TIED_TASK_STACK
2867  // GEH: Figure out if this is the right thing to do
2868  for (i = maxthreads; i < nthreads; i++) {
2869  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
2870  __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data);
2871  }
2872 #endif // BUILD_TIED_TASK_STACK
2873  // Install the new data and free the old data
2874  (*threads_data_p) = new_data;
2875  __kmp_free(old_data);
2876  } else {
2877  KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating "
2878  "threads data for task_team %p, size = %d\n",
2879  __kmp_gtid_from_thread(thread), task_team, nthreads));
2880  // Make the initial allocate for threads_data array, and zero entries
2881  // Cannot use __kmp_thread_calloc() because threads not around for
2882  // kmp_reap_task_team( ).
2883  ANNOTATE_IGNORE_WRITES_BEGIN();
2884  *threads_data_p = (kmp_thread_data_t *)__kmp_allocate(
2885  nthreads * sizeof(kmp_thread_data_t));
2886  ANNOTATE_IGNORE_WRITES_END();
2887 #ifdef BUILD_TIED_TASK_STACK
2888  // GEH: Figure out if this is the right thing to do
2889  for (i = 0; i < nthreads; i++) {
2890  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
2891  __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data);
2892  }
2893 #endif // BUILD_TIED_TASK_STACK
2894  }
2895  task_team->tt.tt_max_threads = nthreads;
2896  } else {
2897  // If array has (more than) enough elements, go ahead and use it
2898  KMP_DEBUG_ASSERT(*threads_data_p != NULL);
2899  }
2900 
2901  // initialize threads_data pointers back to thread_info structures
2902  for (i = 0; i < nthreads; i++) {
2903  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
2904  thread_data->td.td_thr = team->t.t_threads[i];
2905 
2906  if (thread_data->td.td_deque_last_stolen >= nthreads) {
2907  // The last stolen field survives across teams / barrier, and the number
2908  // of threads may have changed. It's possible (likely?) that a new
2909  // parallel region will exhibit the same behavior as previous region.
2910  thread_data->td.td_deque_last_stolen = -1;
2911  }
2912  }
2913 
2914  KMP_MB();
2915  TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE);
2916  }
2917 
2918  __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
2919  return is_init_thread;
2920 }
2921 
2922 // __kmp_free_task_threads_data:
2923 // Deallocates a threads_data array for a task team, including any attached
2924 // tasking deques. Only occurs at library shutdown.
2925 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) {
2926  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
2927  if (task_team->tt.tt_threads_data != NULL) {
2928  int i;
2929  for (i = 0; i < task_team->tt.tt_max_threads; i++) {
2930  __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]);
2931  }
2932  __kmp_free(task_team->tt.tt_threads_data);
2933  task_team->tt.tt_threads_data = NULL;
2934  }
2935  __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
2936 }
2937 
2938 // __kmp_allocate_task_team:
2939 // Allocates a task team associated with a specific team, taking it from
2940 // the global task team free list if possible. Also initializes data
2941 // structures.
2942 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread,
2943  kmp_team_t *team) {
2944  kmp_task_team_t *task_team = NULL;
2945  int nthreads;
2946 
2947  KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n",
2948  (thread ? __kmp_gtid_from_thread(thread) : -1), team));
2949 
2950  if (TCR_PTR(__kmp_free_task_teams) != NULL) {
2951  // Take a task team from the task team pool
2952  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
2953  if (__kmp_free_task_teams != NULL) {
2954  task_team = __kmp_free_task_teams;
2955  TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next);
2956  task_team->tt.tt_next = NULL;
2957  }
2958  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
2959  }
2960 
2961  if (task_team == NULL) {
2962  KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating "
2963  "task team for team %p\n",
2964  __kmp_gtid_from_thread(thread), team));
2965  // Allocate a new task team if one is not available.
2966  // Cannot use __kmp_thread_malloc() because threads not around for
2967  // kmp_reap_task_team( ).
2968  task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t));
2969  __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock);
2970  // AC: __kmp_allocate zeroes returned memory
2971  // task_team -> tt.tt_threads_data = NULL;
2972  // task_team -> tt.tt_max_threads = 0;
2973  // task_team -> tt.tt_next = NULL;
2974  }
2975 
2976  TCW_4(task_team->tt.tt_found_tasks, FALSE);
2977 #if OMP_45_ENABLED
2978  TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
2979 #endif
2980  task_team->tt.tt_nproc = nthreads = team->t.t_nproc;
2981 
2982  TCW_4(task_team->tt.tt_unfinished_threads, nthreads);
2983  TCW_4(task_team->tt.tt_active, TRUE);
2984 
2985  KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
2986  "unfinished_threads init'd to %d\n",
2987  (thread ? __kmp_gtid_from_thread(thread) : -1), task_team,
2988  task_team->tt.tt_unfinished_threads));
2989  return task_team;
2990 }
2991 
2992 // __kmp_free_task_team:
2993 // Frees the task team associated with a specific thread, and adds it
2994 // to the global task team free list.
2995 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) {
2996  KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n",
2997  thread ? __kmp_gtid_from_thread(thread) : -1, task_team));
2998 
2999  // Put task team back on free list
3000  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3001 
3002  KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL);
3003  task_team->tt.tt_next = __kmp_free_task_teams;
3004  TCW_PTR(__kmp_free_task_teams, task_team);
3005 
3006  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3007 }
3008 
3009 // __kmp_reap_task_teams:
3010 // Free all the task teams on the task team free list.
3011 // Should only be done during library shutdown.
3012 // Cannot do anything that needs a thread structure or gtid since they are
3013 // already gone.
3014 void __kmp_reap_task_teams(void) {
3015  kmp_task_team_t *task_team;
3016 
3017  if (TCR_PTR(__kmp_free_task_teams) != NULL) {
3018  // Free all task_teams on the free list
3019  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3020  while ((task_team = __kmp_free_task_teams) != NULL) {
3021  __kmp_free_task_teams = task_team->tt.tt_next;
3022  task_team->tt.tt_next = NULL;
3023 
3024  // Free threads_data if necessary
3025  if (task_team->tt.tt_threads_data != NULL) {
3026  __kmp_free_task_threads_data(task_team);
3027  }
3028  __kmp_free(task_team);
3029  }
3030  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3031  }
3032 }
3033 
3034 // __kmp_wait_to_unref_task_teams:
3035 // Some threads could still be in the fork barrier release code, possibly
3036 // trying to steal tasks. Wait for each thread to unreference its task team.
3037 void __kmp_wait_to_unref_task_teams(void) {
3038  kmp_info_t *thread;
3039  kmp_uint32 spins;
3040  int done;
3041 
3042  KMP_INIT_YIELD(spins);
3043 
3044  for (;;) {
3045  done = TRUE;
3046 
3047  // TODO: GEH - this may be is wrong because some sync would be necessary
3048  // in case threads are added to the pool during the traversal. Need to
3049  // verify that lock for thread pool is held when calling this routine.
3050  for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL;
3051  thread = thread->th.th_next_pool) {
3052 #if KMP_OS_WINDOWS
3053  DWORD exit_val;
3054 #endif
3055  if (TCR_PTR(thread->th.th_task_team) == NULL) {
3056  KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
3057  __kmp_gtid_from_thread(thread)));
3058  continue;
3059  }
3060 #if KMP_OS_WINDOWS
3061  // TODO: GEH - add this check for Linux* OS / OS X* as well?
3062  if (!__kmp_is_thread_alive(thread, &exit_val)) {
3063  thread->th.th_task_team = NULL;
3064  continue;
3065  }
3066 #endif
3067 
3068  done = FALSE; // Because th_task_team pointer is not NULL for this thread
3069 
3070  KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to "
3071  "unreference task_team\n",
3072  __kmp_gtid_from_thread(thread)));
3073 
3074  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
3075  volatile void *sleep_loc;
3076  // If the thread is sleeping, awaken it.
3077  if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) !=
3078  NULL) {
3079  KA_TRACE(
3080  10,
3081  ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",
3082  __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread)));
3083  __kmp_null_resume_wrapper(__kmp_gtid_from_thread(thread), sleep_loc);
3084  }
3085  }
3086  }
3087  if (done) {
3088  break;
3089  }
3090 
3091  // If we are oversubscribed, or have waited a bit (and library mode is
3092  // throughput), yield. Pause is in the following code.
3093  KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
3094  KMP_YIELD_SPIN(spins); // Yields only if KMP_LIBRARY=throughput
3095  }
3096 }
3097 
3098 // __kmp_task_team_setup: Create a task_team for the current team, but use
3099 // an already created, unused one if it already exists.
3100 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
3101  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3102 
3103  // If this task_team hasn't been created yet, allocate it. It will be used in
3104  // the region after the next.
3105  // If it exists, it is the current task team and shouldn't be touched yet as
3106  // it may still be in use.
3107  if (team->t.t_task_team[this_thr->th.th_task_state] == NULL &&
3108  (always || team->t.t_nproc > 1)) {
3109  team->t.t_task_team[this_thr->th.th_task_state] =
3110  __kmp_allocate_task_team(this_thr, team);
3111  KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created new task_team %p "
3112  "for team %d at parity=%d\n",
3113  __kmp_gtid_from_thread(this_thr),
3114  team->t.t_task_team[this_thr->th.th_task_state],
3115  ((team != NULL) ? team->t.t_id : -1),
3116  this_thr->th.th_task_state));
3117  }
3118 
3119  // After threads exit the release, they will call sync, and then point to this
3120  // other task_team; make sure it is allocated and properly initialized. As
3121  // threads spin in the barrier release phase, they will continue to use the
3122  // previous task_team struct(above), until they receive the signal to stop
3123  // checking for tasks (they can't safely reference the kmp_team_t struct,
3124  // which could be reallocated by the master thread). No task teams are formed
3125  // for serialized teams.
3126  if (team->t.t_nproc > 1) {
3127  int other_team = 1 - this_thr->th.th_task_state;
3128  if (team->t.t_task_team[other_team] == NULL) { // setup other team as well
3129  team->t.t_task_team[other_team] =
3130  __kmp_allocate_task_team(this_thr, team);
3131  KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d created second new "
3132  "task_team %p for team %d at parity=%d\n",
3133  __kmp_gtid_from_thread(this_thr),
3134  team->t.t_task_team[other_team],
3135  ((team != NULL) ? team->t.t_id : -1), other_team));
3136  } else { // Leave the old task team struct in place for the upcoming region;
3137  // adjust as needed
3138  kmp_task_team_t *task_team = team->t.t_task_team[other_team];
3139  if (!task_team->tt.tt_active ||
3140  team->t.t_nproc != task_team->tt.tt_nproc) {
3141  TCW_4(task_team->tt.tt_nproc, team->t.t_nproc);
3142  TCW_4(task_team->tt.tt_found_tasks, FALSE);
3143 #if OMP_45_ENABLED
3144  TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3145 #endif
3146  TCW_4(task_team->tt.tt_unfinished_threads, team->t.t_nproc);
3147  TCW_4(task_team->tt.tt_active, TRUE);
3148  }
3149  // if team size has changed, the first thread to enable tasking will
3150  // realloc threads_data if necessary
3151  KA_TRACE(20, ("__kmp_task_team_setup: Master T#%d reset next task_team "
3152  "%p for team %d at parity=%d\n",
3153  __kmp_gtid_from_thread(this_thr),
3154  team->t.t_task_team[other_team],
3155  ((team != NULL) ? team->t.t_id : -1), other_team));
3156  }
3157  }
3158 }
3159 
3160 // __kmp_task_team_sync: Propagation of task team data from team to threads
3161 // which happens just after the release phase of a team barrier. This may be
3162 // called by any thread, but only for teams with # threads > 1.
3163 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
3164  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3165 
3166  // Toggle the th_task_state field, to switch which task_team this thread
3167  // refers to
3168  this_thr->th.th_task_state = 1 - this_thr->th.th_task_state;
3169  // It is now safe to propagate the task team pointer from the team struct to
3170  // the current thread.
3171  TCW_PTR(this_thr->th.th_task_team,
3172  team->t.t_task_team[this_thr->th.th_task_state]);
3173  KA_TRACE(20,
3174  ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
3175  "%p from Team #%d (parity=%d)\n",
3176  __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team,
3177  ((team != NULL) ? team->t.t_id : -1), this_thr->th.th_task_state));
3178 }
3179 
3180 // __kmp_task_team_wait: Master thread waits for outstanding tasks after the
3181 // barrier gather phase. Only called by master thread if #threads in team > 1 or
3182 // if proxy tasks were created.
3183 //
3184 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off
3185 // by passing in 0 optionally as the last argument. When wait is zero, master
3186 // thread does not wait for unfinished_threads to reach 0.
3187 void __kmp_task_team_wait(
3188  kmp_info_t *this_thr,
3189  kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) {
3190  kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state];
3191 
3192  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3193  KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team);
3194 
3195  if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) {
3196  if (wait) {
3197  KA_TRACE(20, ("__kmp_task_team_wait: Master T#%d waiting for all tasks "
3198  "(for unfinished_threads to reach 0) on task_team = %p\n",
3199  __kmp_gtid_from_thread(this_thr), task_team));
3200  // Worker threads may have dropped through to release phase, but could
3201  // still be executing tasks. Wait here for tasks to complete. To avoid
3202  // memory contention, only master thread checks termination condition.
3203  kmp_flag_32 flag(
3204  RCAST(volatile kmp_uint32 *, &task_team->tt.tt_unfinished_threads),
3205  0U);
3206  flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
3207  }
3208  // Deactivate the old task team, so that the worker threads will stop
3209  // referencing it while spinning.
3210  KA_TRACE(
3211  20,
3212  ("__kmp_task_team_wait: Master T#%d deactivating task_team %p: "
3213  "setting active to false, setting local and team's pointer to NULL\n",
3214  __kmp_gtid_from_thread(this_thr), task_team));
3215 #if OMP_45_ENABLED
3216  KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 ||
3217  task_team->tt.tt_found_proxy_tasks == TRUE);
3218  TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3219 #else
3220  KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1);
3221 #endif
3222  KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0);
3223  TCW_SYNC_4(task_team->tt.tt_active, FALSE);
3224  KMP_MB();
3225 
3226  TCW_PTR(this_thr->th.th_task_team, NULL);
3227  }
3228 }
3229 
3230 // __kmp_tasking_barrier:
3231 // This routine may only called when __kmp_tasking_mode == tskm_extra_barrier.
3232 // Internal function to execute all tasks prior to a regular barrier or a join
3233 // barrier. It is a full barrier itself, which unfortunately turns regular
3234 // barriers into double barriers and join barriers into 1 1/2 barriers.
3235 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) {
3236  volatile kmp_uint32 *spin = RCAST(
3237  volatile kmp_uint32 *,
3238  &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads);
3239  int flag = FALSE;
3240  KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier);
3241 
3242 #if USE_ITT_BUILD
3243  KMP_FSYNC_SPIN_INIT(spin, (kmp_uint32 *)NULL);
3244 #endif /* USE_ITT_BUILD */
3245  kmp_flag_32 spin_flag(spin, 0U);
3246  while (!spin_flag.execute_tasks(thread, gtid, TRUE,
3247  &flag USE_ITT_BUILD_ARG(NULL), 0)) {
3248 #if USE_ITT_BUILD
3249  // TODO: What about itt_sync_obj??
3250  KMP_FSYNC_SPIN_PREPARE(CCAST(kmp_uint32 *, spin));
3251 #endif /* USE_ITT_BUILD */
3252 
3253  if (TCR_4(__kmp_global.g.g_done)) {
3254  if (__kmp_global.g.g_abort)
3255  __kmp_abort_thread();
3256  break;
3257  }
3258  KMP_YIELD(TRUE); // GH: We always yield here
3259  }
3260 #if USE_ITT_BUILD
3261  KMP_FSYNC_SPIN_ACQUIRED(CCAST(kmp_uint32 *, spin));
3262 #endif /* USE_ITT_BUILD */
3263 }
3264 
3265 #if OMP_45_ENABLED
3266 
3267 // __kmp_give_task puts a task into a given thread queue if:
3268 // - the queue for that thread was created
3269 // - there's space in that queue
3270 // Because of this, __kmp_push_task needs to check if there's space after
3271 // getting the lock
3272 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task,
3273  kmp_int32 pass) {
3274  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
3275  kmp_task_team_t *task_team = taskdata->td_task_team;
3276 
3277  KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n",
3278  taskdata, tid));
3279 
3280  // If task_team is NULL something went really bad...
3281  KMP_DEBUG_ASSERT(task_team != NULL);
3282 
3283  bool result = false;
3284  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
3285 
3286  if (thread_data->td.td_deque == NULL) {
3287  // There's no queue in this thread, go find another one
3288  // We're guaranteed that at least one thread has a queue
3289  KA_TRACE(30,
3290  ("__kmp_give_task: thread %d has no queue while giving task %p.\n",
3291  tid, taskdata));
3292  return result;
3293  }
3294 
3295  if (TCR_4(thread_data->td.td_deque_ntasks) >=
3296  TASK_DEQUE_SIZE(thread_data->td)) {
3297  KA_TRACE(
3298  30,
3299  ("__kmp_give_task: queue is full while giving task %p to thread %d.\n",
3300  taskdata, tid));
3301 
3302  // if this deque is bigger than the pass ratio give a chance to another
3303  // thread
3304  if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
3305  return result;
3306 
3307  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3308  __kmp_realloc_task_deque(thread, thread_data);
3309 
3310  } else {
3311 
3312  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3313 
3314  if (TCR_4(thread_data->td.td_deque_ntasks) >=
3315  TASK_DEQUE_SIZE(thread_data->td)) {
3316  KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to "
3317  "thread %d.\n",
3318  taskdata, tid));
3319 
3320  // if this deque is bigger than the pass ratio give a chance to another
3321  // thread
3322  if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
3323  goto release_and_exit;
3324 
3325  __kmp_realloc_task_deque(thread, thread_data);
3326  }
3327  }
3328 
3329  // lock is held here, and there is space in the deque
3330 
3331  thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata;
3332  // Wrap index.
3333  thread_data->td.td_deque_tail =
3334  (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
3335  TCW_4(thread_data->td.td_deque_ntasks,
3336  TCR_4(thread_data->td.td_deque_ntasks) + 1);
3337 
3338  result = true;
3339  KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n",
3340  taskdata, tid));
3341 
3342 release_and_exit:
3343  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3344 
3345  return result;
3346 }
3347 
3348 /* The finish of the proxy tasks is divided in two pieces:
3349  - the top half is the one that can be done from a thread outside the team
3350  - the bottom half must be run from a them within the team
3351 
3352  In order to run the bottom half the task gets queued back into one of the
3353  threads of the team. Once the td_incomplete_child_task counter of the parent
3354  is decremented the threads can leave the barriers. So, the bottom half needs
3355  to be queued before the counter is decremented. The top half is therefore
3356  divided in two parts:
3357  - things that can be run before queuing the bottom half
3358  - things that must be run after queuing the bottom half
3359 
3360  This creates a second race as the bottom half can free the task before the
3361  second top half is executed. To avoid this we use the
3362  td_incomplete_child_task of the proxy task to synchronize the top and bottom
3363  half. */
3364 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
3365  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
3366  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3367  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
3368  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
3369 
3370  taskdata->td_flags.complete = 1; // mark the task as completed
3371 
3372  if (taskdata->td_taskgroup)
3373  KMP_TEST_THEN_DEC32(&taskdata->td_taskgroup->count);
3374 
3375  // Create an imaginary children for this task so the bottom half cannot
3376  // release the task before we have completed the second top half
3377  TCI_4(taskdata->td_incomplete_child_tasks);
3378 }
3379 
3380 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
3381  kmp_int32 children = 0;
3382 
3383  // Predecrement simulated by "- 1" calculation
3384  children =
3385  KMP_TEST_THEN_DEC32(&taskdata->td_parent->td_incomplete_child_tasks) - 1;
3386  KMP_DEBUG_ASSERT(children >= 0);
3387 
3388  // Remove the imaginary children
3389  TCD_4(taskdata->td_incomplete_child_tasks);
3390 }
3391 
3392 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) {
3393  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3394  kmp_info_t *thread = __kmp_threads[gtid];
3395 
3396  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3397  KMP_DEBUG_ASSERT(taskdata->td_flags.complete ==
3398  1); // top half must run before bottom half
3399 
3400  // We need to wait to make sure the top half is finished
3401  // Spinning here should be ok as this should happen quickly
3402  while (TCR_4(taskdata->td_incomplete_child_tasks) > 0)
3403  ;
3404 
3405  __kmp_release_deps(gtid, taskdata);
3406  __kmp_free_task_and_ancestors(gtid, taskdata, thread);
3407 }
3408 
3417 void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) {
3418  KMP_DEBUG_ASSERT(ptask != NULL);
3419  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3420  KA_TRACE(
3421  10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n",
3422  gtid, taskdata));
3423 
3424  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3425 
3426  __kmp_first_top_half_finish_proxy(taskdata);
3427  __kmp_second_top_half_finish_proxy(taskdata);
3428  __kmp_bottom_half_finish_proxy(gtid, ptask);
3429 
3430  KA_TRACE(10,
3431  ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n",
3432  gtid, taskdata));
3433 }
3434 
3442 void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) {
3443  KMP_DEBUG_ASSERT(ptask != NULL);
3444  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
3445 
3446  KA_TRACE(
3447  10,
3448  ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n",
3449  taskdata));
3450 
3451  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
3452 
3453  __kmp_first_top_half_finish_proxy(taskdata);
3454 
3455  // Enqueue task to complete bottom half completion from a thread within the
3456  // corresponding team
3457  kmp_team_t *team = taskdata->td_team;
3458  kmp_int32 nthreads = team->t.t_nproc;
3459  kmp_info_t *thread;
3460 
3461  // This should be similar to start_k = __kmp_get_random( thread ) % nthreads
3462  // but we cannot use __kmp_get_random here
3463  kmp_int32 start_k = 0;
3464  kmp_int32 pass = 1;
3465  kmp_int32 k = start_k;
3466 
3467  do {
3468  // For now we're just linearly trying to find a thread
3469  thread = team->t.t_threads[k];
3470  k = (k + 1) % nthreads;
3471 
3472  // we did a full pass through all the threads
3473  if (k == start_k)
3474  pass = pass << 1;
3475 
3476  } while (!__kmp_give_task(thread, k, ptask, pass));
3477 
3478  __kmp_second_top_half_finish_proxy(taskdata);
3479 
3480  KA_TRACE(
3481  10,
3482  ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n",
3483  taskdata));
3484 }
3485 
3486 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task
3487 // for taskloop
3488 //
3489 // thread: allocating thread
3490 // task_src: pointer to source task to be duplicated
3491 // returns: a pointer to the allocated kmp_task_t structure (task).
3492 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
3493  kmp_task_t *task;
3494  kmp_taskdata_t *taskdata;
3495  kmp_taskdata_t *taskdata_src;
3496  kmp_taskdata_t *parent_task = thread->th.th_current_task;
3497  size_t shareds_offset;
3498  size_t task_size;
3499 
3500  KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread,
3501  task_src));
3502  taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
3503  KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy ==
3504  TASK_FULL); // it should not be proxy task
3505  KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT);
3506  task_size = taskdata_src->td_size_alloc;
3507 
3508  // Allocate a kmp_taskdata_t block and a kmp_task_t block.
3509  KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread,
3510  task_size));
3511 #if USE_FAST_MEMORY
3512  taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size);
3513 #else
3514  taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size);
3515 #endif /* USE_FAST_MEMORY */
3516  KMP_MEMCPY(taskdata, taskdata_src, task_size);
3517 
3518  task = KMP_TASKDATA_TO_TASK(taskdata);
3519 
3520  // Initialize new task (only specific fields not affected by memcpy)
3521  taskdata->td_task_id = KMP_GEN_TASK_ID();
3522  if (task->shareds != NULL) { // need setup shareds pointer
3523  shareds_offset = (char *)task_src->shareds - (char *)taskdata_src;
3524  task->shareds = &((char *)taskdata)[shareds_offset];
3525  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
3526  0);
3527  }
3528  taskdata->td_alloc_thread = thread;
3529  taskdata->td_parent = parent_task;
3530  taskdata->td_taskgroup =
3531  parent_task
3532  ->td_taskgroup; // task inherits the taskgroup from the parent task
3533 
3534  // Only need to keep track of child task counts if team parallel and tasking
3535  // not serialized
3536  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
3537  KMP_TEST_THEN_INC32(&parent_task->td_incomplete_child_tasks);
3538  if (parent_task->td_taskgroup)
3539  KMP_TEST_THEN_INC32(&parent_task->td_taskgroup->count);
3540  // Only need to keep track of allocated child tasks for explicit tasks since
3541  // implicit not deallocated
3542  if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT)
3543  KMP_TEST_THEN_INC32(&taskdata->td_parent->td_allocated_child_tasks);
3544  }
3545 
3546  KA_TRACE(20,
3547  ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n",
3548  thread, taskdata, taskdata->td_parent));
3549 #if OMPT_SUPPORT
3550  if (UNLIKELY(ompt_enabled.enabled))
3551  __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid);
3552 #endif
3553  return task;
3554 }
3555 
3556 // Routine optionally generated by the compiler for setting the lastprivate flag
3557 // and calling needed constructors for private/firstprivate objects
3558 // (used to form taskloop tasks from pattern task)
3559 // Parameters: dest task, src task, lastprivate flag.
3560 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
3561 
3562 KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8);
3563 
3564 // class to encapsulate manipulating loop bounds in a taskloop task.
3565 // this abstracts away the Intel vs GOMP taskloop interface for setting/getting
3566 // the loop bound variables.
3567 class kmp_taskloop_bounds_t {
3568  kmp_task_t *task;
3569  const kmp_taskdata_t *taskdata;
3570  size_t lower_offset;
3571  size_t upper_offset;
3572 
3573 public:
3574  kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub)
3575  : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)),
3576  lower_offset((char *)lb - (char *)task),
3577  upper_offset((char *)ub - (char *)task) {
3578  KMP_DEBUG_ASSERT((char *)lb > (char *)_task);
3579  KMP_DEBUG_ASSERT((char *)ub > (char *)_task);
3580  }
3581  kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds)
3582  : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)),
3583  lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {}
3584  size_t get_lower_offset() const { return lower_offset; }
3585  size_t get_upper_offset() const { return upper_offset; }
3586  kmp_uint64 get_lb() const {
3587  kmp_int64 retval;
3588 #if defined(KMP_GOMP_COMPAT)
3589  // Intel task just returns the lower bound normally
3590  if (!taskdata->td_flags.native) {
3591  retval = *(kmp_int64 *)((char *)task + lower_offset);
3592  } else {
3593  // GOMP task has to take into account the sizeof(long)
3594  if (taskdata->td_size_loop_bounds == 4) {
3595  kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds);
3596  retval = (kmp_int64)*lb;
3597  } else {
3598  kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds);
3599  retval = (kmp_int64)*lb;
3600  }
3601  }
3602 #else
3603  retval = *(kmp_int64 *)((char *)task + lower_offset);
3604 #endif // defined(KMP_GOMP_COMPAT)
3605  return retval;
3606  }
3607  kmp_uint64 get_ub() const {
3608  kmp_int64 retval;
3609 #if defined(KMP_GOMP_COMPAT)
3610  // Intel task just returns the upper bound normally
3611  if (!taskdata->td_flags.native) {
3612  retval = *(kmp_int64 *)((char *)task + upper_offset);
3613  } else {
3614  // GOMP task has to take into account the sizeof(long)
3615  if (taskdata->td_size_loop_bounds == 4) {
3616  kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1;
3617  retval = (kmp_int64)*ub;
3618  } else {
3619  kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1;
3620  retval = (kmp_int64)*ub;
3621  }
3622  }
3623 #else
3624  retval = *(kmp_int64 *)((char *)task + upper_offset);
3625 #endif // defined(KMP_GOMP_COMPAT)
3626  return retval;
3627  }
3628  void set_lb(kmp_uint64 lb) {
3629 #if defined(KMP_GOMP_COMPAT)
3630  // Intel task just sets the lower bound normally
3631  if (!taskdata->td_flags.native) {
3632  *(kmp_uint64 *)((char *)task + lower_offset) = lb;
3633  } else {
3634  // GOMP task has to take into account the sizeof(long)
3635  if (taskdata->td_size_loop_bounds == 4) {
3636  kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds);
3637  *lower = (kmp_uint32)lb;
3638  } else {
3639  kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds);
3640  *lower = (kmp_uint64)lb;
3641  }
3642  }
3643 #else
3644  *(kmp_uint64 *)((char *)task + lower_offset) = lb;
3645 #endif // defined(KMP_GOMP_COMPAT)
3646  }
3647  void set_ub(kmp_uint64 ub) {
3648 #if defined(KMP_GOMP_COMPAT)
3649  // Intel task just sets the upper bound normally
3650  if (!taskdata->td_flags.native) {
3651  *(kmp_uint64 *)((char *)task + upper_offset) = ub;
3652  } else {
3653  // GOMP task has to take into account the sizeof(long)
3654  if (taskdata->td_size_loop_bounds == 4) {
3655  kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1;
3656  *upper = (kmp_uint32)ub;
3657  } else {
3658  kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1;
3659  *upper = (kmp_uint64)ub;
3660  }
3661  }
3662 #else
3663  *(kmp_uint64 *)((char *)task + upper_offset) = ub;
3664 #endif // defined(KMP_GOMP_COMPAT)
3665  }
3666 };
3667 
3668 // __kmp_taskloop_linear: Start tasks of the taskloop linearly
3669 //
3670 // loc Source location information
3671 // gtid Global thread ID
3672 // task Pattern task, exposes the loop iteration range
3673 // lb Pointer to loop lower bound in task structure
3674 // ub Pointer to loop upper bound in task structure
3675 // st Loop stride
3676 // ub_glob Global upper bound (used for lastprivate check)
3677 // num_tasks Number of tasks to execute
3678 // grainsize Number of loop iterations per task
3679 // extras Number of chunks with grainsize+1 iterations
3680 // tc Iterations count
3681 // task_dup Tasks duplication routine
3682 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
3683  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
3684  kmp_uint64 ub_glob, kmp_uint64 num_tasks,
3685  kmp_uint64 grainsize, kmp_uint64 extras,
3686  kmp_uint64 tc, void *task_dup) {
3687  KMP_COUNT_BLOCK(OMP_TASKLOOP);
3688  KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling);
3689  p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
3690  // compiler provides global bounds here
3691  kmp_taskloop_bounds_t task_bounds(task, lb, ub);
3692  kmp_uint64 lower = task_bounds.get_lb();
3693  kmp_uint64 upper = task_bounds.get_ub();
3694  kmp_uint64 i;
3695  kmp_info_t *thread = __kmp_threads[gtid];
3696  kmp_taskdata_t *current_task = thread->th.th_current_task;
3697  kmp_task_t *next_task;
3698  kmp_int32 lastpriv = 0;
3699 
3700  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras);
3701  KMP_DEBUG_ASSERT(num_tasks > extras);
3702  KMP_DEBUG_ASSERT(num_tasks > 0);
3703  KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
3704  "extras %lld, i=%lld,%lld(%d)%lld, dup %p\n",
3705  gtid, num_tasks, grainsize, extras, lower, upper, ub_glob, st,
3706  task_dup));
3707 
3708  // Launch num_tasks tasks, assign grainsize iterations each task
3709  for (i = 0; i < num_tasks; ++i) {
3710  kmp_uint64 chunk_minus_1;
3711  if (extras == 0) {
3712  chunk_minus_1 = grainsize - 1;
3713  } else {
3714  chunk_minus_1 = grainsize;
3715  --extras; // first extras iterations get bigger chunk (grainsize+1)
3716  }
3717  upper = lower + st * chunk_minus_1;
3718  if (i == num_tasks - 1) {
3719  // schedule the last task, set lastprivate flag if needed
3720  if (st == 1) { // most common case
3721  KMP_DEBUG_ASSERT(upper == *ub);
3722  if (upper == ub_glob)
3723  lastpriv = 1;
3724  } else if (st > 0) { // positive loop stride
3725  KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper);
3726  if ((kmp_uint64)st > ub_glob - upper)
3727  lastpriv = 1;
3728  } else { // negative loop stride
3729  KMP_DEBUG_ASSERT(upper + st < *ub);
3730  if (upper - ub_glob < (kmp_uint64)(-st))
3731  lastpriv = 1;
3732  }
3733  }
3734  next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
3735  kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
3736  kmp_taskloop_bounds_t next_task_bounds =
3737  kmp_taskloop_bounds_t(next_task, task_bounds);
3738 
3739  // adjust task-specific bounds
3740  next_task_bounds.set_lb(lower);
3741  if (next_taskdata->td_flags.native) {
3742  next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1));
3743  } else {
3744  next_task_bounds.set_ub(upper);
3745  }
3746  if (ptask_dup != NULL) // set lastprivate flag, construct fistprivates, etc.
3747  ptask_dup(next_task, task, lastpriv);
3748  KA_TRACE(40,
3749  ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
3750  "upper %lld stride %lld, (offsets %p %p)\n",
3751  gtid, i, next_task, lower, upper, st,
3752  next_task_bounds.get_lower_offset(),
3753  next_task_bounds.get_upper_offset()));
3754  __kmp_omp_task(gtid, next_task, true); // schedule new task
3755  lower = upper + st; // adjust lower bound for the next iteration
3756  }
3757  // free the pattern task and exit
3758  __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
3759  // do not execute the pattern task, just do internal bookkeeping
3760  __kmp_task_finish(gtid, task, current_task);
3761 }
3762 
3763 // Structure to keep taskloop parameters for auxiliary task
3764 // kept in the shareds of the task structure.
3765 typedef struct __taskloop_params {
3766  kmp_task_t *task;
3767  kmp_uint64 *lb;
3768  kmp_uint64 *ub;
3769  void *task_dup;
3770  kmp_int64 st;
3771  kmp_uint64 ub_glob;
3772  kmp_uint64 num_tasks;
3773  kmp_uint64 grainsize;
3774  kmp_uint64 extras;
3775  kmp_uint64 tc;
3776  kmp_uint64 num_t_min;
3777 } __taskloop_params_t;
3778 
3779 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *,
3780  kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64,
3781  kmp_uint64, kmp_uint64, kmp_uint64, kmp_uint64,
3782  void *);
3783 
3784 // Execute part of the the taskloop submitted as a task.
3785 int __kmp_taskloop_task(int gtid, void *ptask) {
3786  __taskloop_params_t *p =
3787  (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds;
3788  kmp_task_t *task = p->task;
3789  kmp_uint64 *lb = p->lb;
3790  kmp_uint64 *ub = p->ub;
3791  void *task_dup = p->task_dup;
3792  // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
3793  kmp_int64 st = p->st;
3794  kmp_uint64 ub_glob = p->ub_glob;
3795  kmp_uint64 num_tasks = p->num_tasks;
3796  kmp_uint64 grainsize = p->grainsize;
3797  kmp_uint64 extras = p->extras;
3798  kmp_uint64 tc = p->tc;
3799  kmp_uint64 num_t_min = p->num_t_min;
3800 #if KMP_DEBUG
3801  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
3802  KMP_DEBUG_ASSERT(task != NULL);
3803  KA_TRACE(20, ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
3804  " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n",
3805  gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st,
3806  task_dup));
3807 #endif
3808  KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min);
3809  if (num_tasks > num_t_min)
3810  __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
3811  grainsize, extras, tc, num_t_min, task_dup);
3812  else
3813  __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
3814  grainsize, extras, tc, task_dup);
3815 
3816  KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid));
3817  return 0;
3818 }
3819 
3820 // Schedule part of the the taskloop as a task,
3821 // execute the rest of the the taskloop.
3822 //
3823 // loc Source location information
3824 // gtid Global thread ID
3825 // task Pattern task, exposes the loop iteration range
3826 // lb Pointer to loop lower bound in task structure
3827 // ub Pointer to loop upper bound in task structure
3828 // st Loop stride
3829 // ub_glob Global upper bound (used for lastprivate check)
3830 // num_tasks Number of tasks to execute
3831 // grainsize Number of loop iterations per task
3832 // extras Number of chunks with grainsize+1 iterations
3833 // tc Iterations count
3834 // num_t_min Threashold to launch tasks recursively
3835 // task_dup Tasks duplication routine
3836 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
3837  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
3838  kmp_uint64 ub_glob, kmp_uint64 num_tasks,
3839  kmp_uint64 grainsize, kmp_uint64 extras,
3840  kmp_uint64 tc, kmp_uint64 num_t_min, void *task_dup) {
3841 #if KMP_DEBUG
3842  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
3843  KMP_DEBUG_ASSERT(task != NULL);
3844  KMP_DEBUG_ASSERT(num_tasks > num_t_min);
3845  KA_TRACE(20, ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
3846  " %lld, extras %lld, i=%lld,%lld(%d), dup %p\n",
3847  gtid, taskdata, num_tasks, grainsize, extras, *lb, *ub, st,
3848  task_dup));
3849 #endif
3850  p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
3851  kmp_uint64 lower = *lb;
3852  kmp_uint64 upper = *ub;
3853  kmp_info_t *thread = __kmp_threads[gtid];
3854  // kmp_taskdata_t *current_task = thread->th.th_current_task;
3855  kmp_task_t *next_task;
3856  kmp_int32 lastpriv = 0;
3857  size_t lower_offset =
3858  (char *)lb - (char *)task; // remember offset of lb in the task structure
3859  size_t upper_offset =
3860  (char *)ub - (char *)task; // remember offset of ub in the task structure
3861 
3862  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras);
3863  KMP_DEBUG_ASSERT(num_tasks > extras);
3864  KMP_DEBUG_ASSERT(num_tasks > 0);
3865 
3866  // split the loop in two halves
3867  kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1;
3868  kmp_uint64 gr_size0 = grainsize;
3869  kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute
3870  kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task
3871  if (n_tsk0 <= extras) {
3872  gr_size0++; // integrate extras into grainsize
3873  ext0 = 0; // no extra iters in 1st half
3874  ext1 = extras - n_tsk0; // remaining extras
3875  tc0 = gr_size0 * n_tsk0;
3876  tc1 = tc - tc0;
3877  } else { // n_tsk0 > extras
3878  ext1 = 0; // no extra iters in 2nd half
3879  ext0 = extras;
3880  tc1 = grainsize * n_tsk1;
3881  tc0 = tc - tc1;
3882  }
3883  ub0 = lower + st * (tc0 - 1);
3884  lb1 = ub0 + st;
3885 
3886  // create pattern task for 2nd half of the loop
3887  next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
3888  // adjust lower bound (upper bound is not changed) for the 2nd half
3889  *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
3890  if (ptask_dup != NULL) // construct fistprivates, etc.
3891  ptask_dup(next_task, task, 0);
3892  *ub = ub0; // adjust upper bound for the 1st half
3893 
3894  // create auxiliary task for 2nd half of the loop
3895  kmp_task_t *new_task =
3896  __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *),
3897  sizeof(__taskloop_params_t), &__kmp_taskloop_task);
3898  __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds;
3899  p->task = next_task;
3900  p->lb = (kmp_uint64 *)((char *)next_task + lower_offset);
3901  p->ub = (kmp_uint64 *)((char *)next_task + upper_offset);
3902  p->task_dup = task_dup;
3903  p->st = st;
3904  p->ub_glob = ub_glob;
3905  p->num_tasks = n_tsk1;
3906  p->grainsize = grainsize;
3907  p->extras = ext1;
3908  p->tc = tc1;
3909  p->num_t_min = num_t_min;
3910  __kmp_omp_task(gtid, new_task, true); // schedule new task
3911 
3912  // execute the 1st half of current subrange
3913  if (n_tsk0 > num_t_min)
3914  __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0,
3915  ext0, tc0, num_t_min, task_dup);
3916  else
3917  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
3918  gr_size0, ext0, tc0, task_dup);
3919 
3920  KA_TRACE(40, ("__kmpc_taskloop_recur(exit): T#%d\n", gtid));
3921 }
3922 
3939 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
3940  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup,
3941  int sched, kmp_uint64 grainsize, void *task_dup) {
3942  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
3943  KMP_DEBUG_ASSERT(task != NULL);
3944 
3945 #if OMPT_SUPPORT && OMPT_OPTIONAL
3946  ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
3947  ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
3948  if (ompt_enabled.ompt_callback_work) {
3949  ompt_callbacks.ompt_callback(ompt_callback_work)(
3950  ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
3951  &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
3952  }
3953 #endif
3954 
3955  if (nogroup == 0) {
3956 #if OMPT_SUPPORT && OMPT_OPTIONAL
3957  OMPT_STORE_RETURN_ADDRESS(gtid);
3958 #endif
3959  __kmpc_taskgroup(loc, gtid);
3960  }
3961 
3962  // =========================================================================
3963  // calculate loop parameters
3964  kmp_taskloop_bounds_t task_bounds(task, lb, ub);
3965  kmp_uint64 tc;
3966  // compiler provides global bounds here
3967  kmp_uint64 lower = task_bounds.get_lb();
3968  kmp_uint64 upper = task_bounds.get_ub();
3969  kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag
3970  kmp_uint64 num_tasks = 0, extras = 0;
3971  kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks;
3972  kmp_info_t *thread = __kmp_threads[gtid];
3973  kmp_taskdata_t *current_task = thread->th.th_current_task;
3974 
3975  KA_TRACE(20, ("__kmpc_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
3976  "grain %llu(%d), dup %p\n",
3977  gtid, taskdata, lower, upper, st, grainsize, sched, task_dup));
3978 
3979  // compute trip count
3980  if (st == 1) { // most common case
3981  tc = upper - lower + 1;
3982  } else if (st < 0) {
3983  tc = (lower - upper) / (-st) + 1;
3984  } else { // st > 0
3985  tc = (upper - lower) / st + 1;
3986  }
3987  if (tc == 0) {
3988  KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d zero-trip loop\n", gtid));
3989  // free the pattern task and exit
3990  __kmp_task_start(gtid, task, current_task);
3991  // do not execute anything for zero-trip loop
3992  __kmp_task_finish(gtid, task, current_task);
3993  return;
3994  }
3995  if (num_tasks_min == 0)
3996  // TODO: can we choose better default heuristic?
3997  num_tasks_min =
3998  KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE);
3999 
4000  // compute num_tasks/grainsize based on the input provided
4001  switch (sched) {
4002  case 0: // no schedule clause specified, we can choose the default
4003  // let's try to schedule (team_size*10) tasks
4004  grainsize = thread->th.th_team_nproc * 10;
4005  case 2: // num_tasks provided
4006  if (grainsize > tc) {
4007  num_tasks = tc; // too big num_tasks requested, adjust values
4008  grainsize = 1;
4009  extras = 0;
4010  } else {
4011  num_tasks = grainsize;
4012  grainsize = tc / num_tasks;
4013  extras = tc % num_tasks;
4014  }
4015  break;
4016  case 1: // grainsize provided
4017  if (grainsize > tc) {
4018  num_tasks = 1; // too big grainsize requested, adjust values
4019  grainsize = tc;
4020  extras = 0;
4021  } else {
4022  num_tasks = tc / grainsize;
4023  // adjust grainsize for balanced distribution of iterations
4024  grainsize = tc / num_tasks;
4025  extras = tc % num_tasks;
4026  }
4027  break;
4028  default:
4029  KMP_ASSERT2(0, "unknown scheduling of taskloop");
4030  }
4031  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize + extras);
4032  KMP_DEBUG_ASSERT(num_tasks > extras);
4033  KMP_DEBUG_ASSERT(num_tasks > 0);
4034  // =========================================================================
4035 
4036  // check if clause value first
4037  // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native)
4038  if (if_val == 0) { // if(0) specified, mark task as serial
4039  taskdata->td_flags.task_serial = 1;
4040  taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
4041 #if OMPT_SUPPORT && OMPT_OPTIONAL
4042  OMPT_STORE_RETURN_ADDRESS(gtid);
4043 #endif
4044  // always start serial tasks linearly
4045  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4046  grainsize, extras, tc, task_dup);
4047  // !taskdata->td_flags.native => currently force linear spawning of tasks
4048  // for GOMP_taskloop
4049  } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) {
4050  KA_TRACE(20, ("__kmpc_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
4051  "(%lld), grain %llu, extras %llu\n",
4052  gtid, tc, num_tasks, num_tasks_min, grainsize, extras));
4053 #if OMPT_SUPPORT && OMPT_OPTIONAL
4054  OMPT_STORE_RETURN_ADDRESS(gtid);
4055 #endif
4056  __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4057  grainsize, extras, tc, num_tasks_min, task_dup);
4058  } else {
4059  KA_TRACE(20, ("__kmpc_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
4060  "(%lld), grain %llu, extras %llu\n",
4061  gtid, tc, num_tasks, num_tasks_min, grainsize, extras));
4062 #if OMPT_SUPPORT && OMPT_OPTIONAL
4063  OMPT_STORE_RETURN_ADDRESS(gtid);
4064 #endif
4065  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
4066  grainsize, extras, tc, task_dup);
4067  }
4068 
4069  if (nogroup == 0) {
4070 #if OMPT_SUPPORT && OMPT_OPTIONAL
4071  OMPT_STORE_RETURN_ADDRESS(gtid);
4072 #endif
4073  __kmpc_end_taskgroup(loc, gtid);
4074  }
4075 #if OMPT_SUPPORT && OMPT_OPTIONAL
4076  if (ompt_enabled.ompt_callback_work) {
4077  ompt_callbacks.ompt_callback(ompt_callback_work)(
4078  ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data),
4079  &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
4080  }
4081 #endif
4082  KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid));
4083 }
4084 
4085 #endif
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
Definition: kmp_stats.h:816
Definition: kmp.h:210