LLVM OpenMP* Runtime Library
kmp_csupport.cpp
1 /*
2  * kmp_csupport.cpp -- kfront linkage support for OpenMP.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #define __KMP_IMP
14 #include "omp.h" /* extern "C" declarations of user-visible routines */
15 #include "kmp.h"
16 #include "kmp_error.h"
17 #include "kmp_i18n.h"
18 #include "kmp_itt.h"
19 #include "kmp_lock.h"
20 #include "kmp_stats.h"
21 #include "ompt-specific.h"
22 
23 #define MAX_MESSAGE 512
24 
25 // flags will be used in future, e.g. to implement openmp_strict library
26 // restrictions
27 
36 void __kmpc_begin(ident_t *loc, kmp_int32 flags) {
37  // By default __kmpc_begin() is no-op.
38  char *env;
39  if ((env = getenv("KMP_INITIAL_THREAD_BIND")) != NULL &&
40  __kmp_str_match_true(env)) {
41  __kmp_middle_initialize();
42  __kmp_assign_root_init_mask();
43  KC_TRACE(10, ("__kmpc_begin: middle initialization called\n"));
44  } else if (__kmp_ignore_mppbeg() == FALSE) {
45  // By default __kmp_ignore_mppbeg() returns TRUE.
46  __kmp_internal_begin();
47  KC_TRACE(10, ("__kmpc_begin: called\n"));
48  }
49 }
50 
59 void __kmpc_end(ident_t *loc) {
60  // By default, __kmp_ignore_mppend() returns TRUE which makes __kmpc_end()
61  // call no-op. However, this can be overridden with KMP_IGNORE_MPPEND
62  // environment variable. If KMP_IGNORE_MPPEND is 0, __kmp_ignore_mppend()
63  // returns FALSE and __kmpc_end() will unregister this root (it can cause
64  // library shut down).
65  if (__kmp_ignore_mppend() == FALSE) {
66  KC_TRACE(10, ("__kmpc_end: called\n"));
67  KA_TRACE(30, ("__kmpc_end\n"));
68 
69  __kmp_internal_end_thread(-1);
70  }
71 #if KMP_OS_WINDOWS && OMPT_SUPPORT
72  // Normal exit process on Windows does not allow worker threads of the final
73  // parallel region to finish reporting their events, so shutting down the
74  // library here fixes the issue at least for the cases where __kmpc_end() is
75  // placed properly.
76  if (ompt_enabled.enabled)
77  __kmp_internal_end_library(__kmp_gtid_get_specific());
78 #endif
79 }
80 
99 kmp_int32 __kmpc_global_thread_num(ident_t *loc) {
100  kmp_int32 gtid = __kmp_entry_gtid();
101 
102  KC_TRACE(10, ("__kmpc_global_thread_num: T#%d\n", gtid));
103 
104  return gtid;
105 }
106 
121 kmp_int32 __kmpc_global_num_threads(ident_t *loc) {
122  KC_TRACE(10,
123  ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
124 
125  return TCR_4(__kmp_all_nth);
126 }
127 
134 kmp_int32 __kmpc_bound_thread_num(ident_t *loc) {
135  KC_TRACE(10, ("__kmpc_bound_thread_num: called\n"));
136  return __kmp_tid_from_gtid(__kmp_entry_gtid());
137 }
138 
144 kmp_int32 __kmpc_bound_num_threads(ident_t *loc) {
145  KC_TRACE(10, ("__kmpc_bound_num_threads: called\n"));
146 
147  return __kmp_entry_thread()->th.th_team->t.t_nproc;
148 }
149 
156 kmp_int32 __kmpc_ok_to_fork(ident_t *loc) {
157 #ifndef KMP_DEBUG
158 
159  return TRUE;
160 
161 #else
162 
163  const char *semi2;
164  const char *semi3;
165  int line_no;
166 
167  if (__kmp_par_range == 0) {
168  return TRUE;
169  }
170  semi2 = loc->psource;
171  if (semi2 == NULL) {
172  return TRUE;
173  }
174  semi2 = strchr(semi2, ';');
175  if (semi2 == NULL) {
176  return TRUE;
177  }
178  semi2 = strchr(semi2 + 1, ';');
179  if (semi2 == NULL) {
180  return TRUE;
181  }
182  if (__kmp_par_range_filename[0]) {
183  const char *name = semi2 - 1;
184  while ((name > loc->psource) && (*name != '/') && (*name != ';')) {
185  name--;
186  }
187  if ((*name == '/') || (*name == ';')) {
188  name++;
189  }
190  if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
191  return __kmp_par_range < 0;
192  }
193  }
194  semi3 = strchr(semi2 + 1, ';');
195  if (__kmp_par_range_routine[0]) {
196  if ((semi3 != NULL) && (semi3 > semi2) &&
197  (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
198  return __kmp_par_range < 0;
199  }
200  }
201  if (KMP_SSCANF(semi3 + 1, "%d", &line_no) == 1) {
202  if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
203  return __kmp_par_range > 0;
204  }
205  return __kmp_par_range < 0;
206  }
207  return TRUE;
208 
209 #endif /* KMP_DEBUG */
210 }
211 
218 kmp_int32 __kmpc_in_parallel(ident_t *loc) {
219  return __kmp_entry_thread()->th.th_root->r.r_active;
220 }
221 
231 void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
232  kmp_int32 num_threads) {
233  KA_TRACE(20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
234  global_tid, num_threads));
235  __kmp_assert_valid_gtid(global_tid);
236  __kmp_push_num_threads(loc, global_tid, num_threads);
237 }
238 
239 void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid) {
240  KA_TRACE(20, ("__kmpc_pop_num_threads: enter\n"));
241  /* the num_threads are automatically popped */
242 }
243 
244 void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
245  kmp_int32 proc_bind) {
246  KA_TRACE(20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
247  proc_bind));
248  __kmp_assert_valid_gtid(global_tid);
249  __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
250 }
251 
262 void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...) {
263  int gtid = __kmp_entry_gtid();
264 
265 #if (KMP_STATS_ENABLED)
266  // If we were in a serial region, then stop the serial timer, record
267  // the event, and start parallel region timer
268  stats_state_e previous_state = KMP_GET_THREAD_STATE();
269  if (previous_state == stats_state_e::SERIAL_REGION) {
270  KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
271  } else {
272  KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead);
273  }
274  int inParallel = __kmpc_in_parallel(loc);
275  if (inParallel) {
276  KMP_COUNT_BLOCK(OMP_NESTED_PARALLEL);
277  } else {
278  KMP_COUNT_BLOCK(OMP_PARALLEL);
279  }
280 #endif
281 
282  // maybe to save thr_state is enough here
283  {
284  va_list ap;
285  va_start(ap, microtask);
286 
287 #if OMPT_SUPPORT
288  ompt_frame_t *ompt_frame;
289  if (ompt_enabled.enabled) {
290  kmp_info_t *master_th = __kmp_threads[gtid];
291  kmp_team_t *parent_team = master_th->th.th_team;
292  ompt_lw_taskteam_t *lwt = parent_team->t.ompt_serialized_team_info;
293  if (lwt)
294  ompt_frame = &(lwt->ompt_task_info.frame);
295  else {
296  int tid = __kmp_tid_from_gtid(gtid);
297  ompt_frame = &(
298  parent_team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame);
299  }
300  ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
301  }
302  OMPT_STORE_RETURN_ADDRESS(gtid);
303 #endif
304 
305 #if INCLUDE_SSC_MARKS
306  SSC_MARK_FORKING();
307 #endif
308  __kmp_fork_call(loc, gtid, fork_context_intel, argc,
309  VOLATILE_CAST(microtask_t) microtask, // "wrapped" task
310  VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
311  kmp_va_addr_of(ap));
312 #if INCLUDE_SSC_MARKS
313  SSC_MARK_JOINING();
314 #endif
315  __kmp_join_call(loc, gtid
316 #if OMPT_SUPPORT
317  ,
318  fork_context_intel
319 #endif
320  );
321 
322  va_end(ap);
323  }
324 
325 #if KMP_STATS_ENABLED
326  if (previous_state == stats_state_e::SERIAL_REGION) {
327  KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
328  KMP_SET_THREAD_STATE(previous_state);
329  } else {
330  KMP_POP_PARTITIONED_TIMER();
331  }
332 #endif // KMP_STATS_ENABLED
333 }
334 
346 void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
347  kmp_int32 num_teams, kmp_int32 num_threads) {
348  KA_TRACE(20,
349  ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
350  global_tid, num_teams, num_threads));
351  __kmp_assert_valid_gtid(global_tid);
352  __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
353 }
354 
371 void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid,
372  kmp_int32 num_teams_lb, kmp_int32 num_teams_ub,
373  kmp_int32 num_threads) {
374  KA_TRACE(20, ("__kmpc_push_num_teams_51: enter T#%d num_teams_lb=%d"
375  " num_teams_ub=%d num_threads=%d\n",
376  global_tid, num_teams_lb, num_teams_ub, num_threads));
377  __kmp_assert_valid_gtid(global_tid);
378  __kmp_push_num_teams_51(loc, global_tid, num_teams_lb, num_teams_ub,
379  num_threads);
380 }
381 
392 void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,
393  ...) {
394  int gtid = __kmp_entry_gtid();
395  kmp_info_t *this_thr = __kmp_threads[gtid];
396  va_list ap;
397  va_start(ap, microtask);
398 
399 #if KMP_STATS_ENABLED
400  KMP_COUNT_BLOCK(OMP_TEAMS);
401  stats_state_e previous_state = KMP_GET_THREAD_STATE();
402  if (previous_state == stats_state_e::SERIAL_REGION) {
403  KMP_EXCHANGE_PARTITIONED_TIMER(OMP_teams_overhead);
404  } else {
405  KMP_PUSH_PARTITIONED_TIMER(OMP_teams_overhead);
406  }
407 #endif
408 
409  // remember teams entry point and nesting level
410  this_thr->th.th_teams_microtask = microtask;
411  this_thr->th.th_teams_level =
412  this_thr->th.th_team->t.t_level; // AC: can be >0 on host
413 
414 #if OMPT_SUPPORT
415  kmp_team_t *parent_team = this_thr->th.th_team;
416  int tid = __kmp_tid_from_gtid(gtid);
417  if (ompt_enabled.enabled) {
418  parent_team->t.t_implicit_task_taskdata[tid]
419  .ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
420  }
421  OMPT_STORE_RETURN_ADDRESS(gtid);
422 #endif
423 
424  // check if __kmpc_push_num_teams called, set default number of teams
425  // otherwise
426  if (this_thr->th.th_teams_size.nteams == 0) {
427  __kmp_push_num_teams(loc, gtid, 0, 0);
428  }
429  KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
430  KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
431  KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
432 
433  __kmp_fork_call(
434  loc, gtid, fork_context_intel, argc,
435  VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task
436  VOLATILE_CAST(launch_t) __kmp_invoke_teams_master, kmp_va_addr_of(ap));
437  __kmp_join_call(loc, gtid
438 #if OMPT_SUPPORT
439  ,
440  fork_context_intel
441 #endif
442  );
443 
444  // Pop current CG root off list
445  KMP_DEBUG_ASSERT(this_thr->th.th_cg_roots);
446  kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
447  this_thr->th.th_cg_roots = tmp->up;
448  KA_TRACE(100, ("__kmpc_fork_teams: Thread %p popping node %p and moving up"
449  " to node %p. cg_nthreads was %d\n",
450  this_thr, tmp, this_thr->th.th_cg_roots, tmp->cg_nthreads));
451  KMP_DEBUG_ASSERT(tmp->cg_nthreads);
452  int i = tmp->cg_nthreads--;
453  if (i == 1) { // check is we are the last thread in CG (not always the case)
454  __kmp_free(tmp);
455  }
456  // Restore current task's thread_limit from CG root
457  KMP_DEBUG_ASSERT(this_thr->th.th_cg_roots);
458  this_thr->th.th_current_task->td_icvs.thread_limit =
459  this_thr->th.th_cg_roots->cg_thread_limit;
460 
461  this_thr->th.th_teams_microtask = NULL;
462  this_thr->th.th_teams_level = 0;
463  *(kmp_int64 *)(&this_thr->th.th_teams_size) = 0L;
464  va_end(ap);
465 #if KMP_STATS_ENABLED
466  if (previous_state == stats_state_e::SERIAL_REGION) {
467  KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
468  KMP_SET_THREAD_STATE(previous_state);
469  } else {
470  KMP_POP_PARTITIONED_TIMER();
471  }
472 #endif // KMP_STATS_ENABLED
473 }
474 
475 // I don't think this function should ever have been exported.
476 // The __kmpc_ prefix was misapplied. I'm fairly certain that no generated
477 // openmp code ever called it, but it's been exported from the RTL for so
478 // long that I'm afraid to remove the definition.
479 int __kmpc_invoke_task_func(int gtid) { return __kmp_invoke_task_func(gtid); }
480 
493 void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
494  // The implementation is now in kmp_runtime.cpp so that it can share static
495  // functions with kmp_fork_call since the tasks to be done are similar in
496  // each case.
497  __kmp_assert_valid_gtid(global_tid);
498 #if OMPT_SUPPORT
499  OMPT_STORE_RETURN_ADDRESS(global_tid);
500 #endif
501  __kmp_serialized_parallel(loc, global_tid);
502 }
503 
511 void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
512  kmp_internal_control_t *top;
513  kmp_info_t *this_thr;
514  kmp_team_t *serial_team;
515 
516  KC_TRACE(10,
517  ("__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
518 
519  /* skip all this code for autopar serialized loops since it results in
520  unacceptable overhead */
521  if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
522  return;
523 
524  // Not autopar code
525  __kmp_assert_valid_gtid(global_tid);
526  if (!TCR_4(__kmp_init_parallel))
527  __kmp_parallel_initialize();
528 
529  __kmp_resume_if_soft_paused();
530 
531  this_thr = __kmp_threads[global_tid];
532  serial_team = this_thr->th.th_serial_team;
533 
534  kmp_task_team_t *task_team = this_thr->th.th_task_team;
535  // we need to wait for the proxy tasks before finishing the thread
536  if (task_team != NULL && task_team->tt.tt_found_proxy_tasks)
537  __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
538 
539  KMP_MB();
540  KMP_DEBUG_ASSERT(serial_team);
541  KMP_ASSERT(serial_team->t.t_serialized);
542  KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
543  KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
544  KMP_DEBUG_ASSERT(serial_team->t.t_threads);
545  KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
546 
547 #if OMPT_SUPPORT
548  if (ompt_enabled.enabled &&
549  this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
550  OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = ompt_data_none;
551  if (ompt_enabled.ompt_callback_implicit_task) {
552  ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
553  ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
554  OMPT_CUR_TASK_INFO(this_thr)->thread_num, ompt_task_implicit);
555  }
556 
557  // reset clear the task id only after unlinking the task
558  ompt_data_t *parent_task_data;
559  __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
560 
561  if (ompt_enabled.ompt_callback_parallel_end) {
562  ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
563  &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
564  ompt_parallel_invoker_program | ompt_parallel_team,
565  OMPT_LOAD_RETURN_ADDRESS(global_tid));
566  }
567  __ompt_lw_taskteam_unlink(this_thr);
568  this_thr->th.ompt_thread_info.state = ompt_state_overhead;
569  }
570 #endif
571 
572  /* If necessary, pop the internal control stack values and replace the team
573  * values */
574  top = serial_team->t.t_control_stack_top;
575  if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
576  copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
577  serial_team->t.t_control_stack_top = top->next;
578  __kmp_free(top);
579  }
580 
581  // if( serial_team -> t.t_serialized > 1 )
582  serial_team->t.t_level--;
583 
584  /* pop dispatch buffers stack */
585  KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
586  {
587  dispatch_private_info_t *disp_buffer =
588  serial_team->t.t_dispatch->th_disp_buffer;
589  serial_team->t.t_dispatch->th_disp_buffer =
590  serial_team->t.t_dispatch->th_disp_buffer->next;
591  __kmp_free(disp_buffer);
592  }
593  this_thr->th.th_def_allocator = serial_team->t.t_def_allocator; // restore
594 
595  --serial_team->t.t_serialized;
596  if (serial_team->t.t_serialized == 0) {
597 
598  /* return to the parallel section */
599 
600 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
601  if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
602  __kmp_clear_x87_fpu_status_word();
603  __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
604  __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
605  }
606 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
607 
608 #if OMPD_SUPPORT
609  if (ompd_state & OMPD_ENABLE_BP)
610  ompd_bp_parallel_end();
611 #endif
612 
613  this_thr->th.th_team = serial_team->t.t_parent;
614  this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
615 
616  /* restore values cached in the thread */
617  this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc; /* JPH */
618  this_thr->th.th_team_master =
619  serial_team->t.t_parent->t.t_threads[0]; /* JPH */
620  this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
621 
622  /* TODO the below shouldn't need to be adjusted for serialized teams */
623  this_thr->th.th_dispatch =
624  &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
625 
626  __kmp_pop_current_task_from_thread(this_thr);
627 
628  KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
629  this_thr->th.th_current_task->td_flags.executing = 1;
630 
631  if (__kmp_tasking_mode != tskm_immediate_exec) {
632  // Copy the task team from the new child / old parent team to the thread.
633  this_thr->th.th_task_team =
634  this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
635  KA_TRACE(20,
636  ("__kmpc_end_serialized_parallel: T#%d restoring task_team %p / "
637  "team %p\n",
638  global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
639  }
640  } else {
641  if (__kmp_tasking_mode != tskm_immediate_exec) {
642  KA_TRACE(20, ("__kmpc_end_serialized_parallel: T#%d decreasing nesting "
643  "depth of serial team %p to %d\n",
644  global_tid, serial_team, serial_team->t.t_serialized));
645  }
646  }
647 
648  if (__kmp_env_consistency_check)
649  __kmp_pop_parallel(global_tid, NULL);
650 #if OMPT_SUPPORT
651  if (ompt_enabled.enabled)
652  this_thr->th.ompt_thread_info.state =
653  ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
654  : ompt_state_work_parallel);
655 #endif
656 }
657 
666 void __kmpc_flush(ident_t *loc) {
667  KC_TRACE(10, ("__kmpc_flush: called\n"));
668 
669  /* need explicit __mf() here since use volatile instead in library */
670  KMP_MB(); /* Flush all pending memory write invalidates. */
671 
672 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
673 #if KMP_MIC
674 // fence-style instructions do not exist, but lock; xaddl $0,(%rsp) can be used.
675 // We shouldn't need it, though, since the ABI rules require that
676 // * If the compiler generates NGO stores it also generates the fence
677 // * If users hand-code NGO stores they should insert the fence
678 // therefore no incomplete unordered stores should be visible.
679 #else
680  // C74404
681  // This is to address non-temporal store instructions (sfence needed).
682  // The clflush instruction is addressed either (mfence needed).
683  // Probably the non-temporal load monvtdqa instruction should also be
684  // addressed.
685  // mfence is a SSE2 instruction. Do not execute it if CPU is not SSE2.
686  if (!__kmp_cpuinfo.initialized) {
687  __kmp_query_cpuid(&__kmp_cpuinfo);
688  }
689  if (!__kmp_cpuinfo.sse2) {
690  // CPU cannot execute SSE2 instructions.
691  } else {
692 #if KMP_COMPILER_ICC
693  _mm_mfence();
694 #elif KMP_COMPILER_MSVC
695  MemoryBarrier();
696 #else
697  __sync_synchronize();
698 #endif // KMP_COMPILER_ICC
699  }
700 #endif // KMP_MIC
701 #elif (KMP_ARCH_ARM || KMP_ARCH_AARCH64 || KMP_ARCH_MIPS || KMP_ARCH_MIPS64 || \
702  KMP_ARCH_RISCV64)
703 // Nothing to see here move along
704 #elif KMP_ARCH_PPC64
705 // Nothing needed here (we have a real MB above).
706 #else
707 #error Unknown or unsupported architecture
708 #endif
709 
710 #if OMPT_SUPPORT && OMPT_OPTIONAL
711  if (ompt_enabled.ompt_callback_flush) {
712  ompt_callbacks.ompt_callback(ompt_callback_flush)(
713  __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
714  }
715 #endif
716 }
717 
718 /* -------------------------------------------------------------------------- */
726 void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) {
727  KMP_COUNT_BLOCK(OMP_BARRIER);
728  KC_TRACE(10, ("__kmpc_barrier: called T#%d\n", global_tid));
729  __kmp_assert_valid_gtid(global_tid);
730 
731  if (!TCR_4(__kmp_init_parallel))
732  __kmp_parallel_initialize();
733 
734  __kmp_resume_if_soft_paused();
735 
736  if (__kmp_env_consistency_check) {
737  if (loc == 0) {
738  KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
739  }
740  __kmp_check_barrier(global_tid, ct_barrier, loc);
741  }
742 
743 #if OMPT_SUPPORT
744  ompt_frame_t *ompt_frame;
745  if (ompt_enabled.enabled) {
746  __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
747  if (ompt_frame->enter_frame.ptr == NULL)
748  ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
749  }
750  OMPT_STORE_RETURN_ADDRESS(global_tid);
751 #endif
752  __kmp_threads[global_tid]->th.th_ident = loc;
753  // TODO: explicit barrier_wait_id:
754  // this function is called when 'barrier' directive is present or
755  // implicit barrier at the end of a worksharing construct.
756  // 1) better to add a per-thread barrier counter to a thread data structure
757  // 2) set to 0 when a new team is created
758  // 4) no sync is required
759 
760  __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
761 #if OMPT_SUPPORT && OMPT_OPTIONAL
762  if (ompt_enabled.enabled) {
763  ompt_frame->enter_frame = ompt_data_none;
764  }
765 #endif
766 }
767 
768 /* The BARRIER for a MASTER section is always explicit */
775 kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid) {
776  int status = 0;
777 
778  KC_TRACE(10, ("__kmpc_master: called T#%d\n", global_tid));
779  __kmp_assert_valid_gtid(global_tid);
780 
781  if (!TCR_4(__kmp_init_parallel))
782  __kmp_parallel_initialize();
783 
784  __kmp_resume_if_soft_paused();
785 
786  if (KMP_MASTER_GTID(global_tid)) {
787  KMP_COUNT_BLOCK(OMP_MASTER);
788  KMP_PUSH_PARTITIONED_TIMER(OMP_master);
789  status = 1;
790  }
791 
792 #if OMPT_SUPPORT && OMPT_OPTIONAL
793  if (status) {
794  if (ompt_enabled.ompt_callback_masked) {
795  kmp_info_t *this_thr = __kmp_threads[global_tid];
796  kmp_team_t *team = this_thr->th.th_team;
797 
798  int tid = __kmp_tid_from_gtid(global_tid);
799  ompt_callbacks.ompt_callback(ompt_callback_masked)(
800  ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
801  &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
802  OMPT_GET_RETURN_ADDRESS(0));
803  }
804  }
805 #endif
806 
807  if (__kmp_env_consistency_check) {
808 #if KMP_USE_DYNAMIC_LOCK
809  if (status)
810  __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
811  else
812  __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
813 #else
814  if (status)
815  __kmp_push_sync(global_tid, ct_master, loc, NULL);
816  else
817  __kmp_check_sync(global_tid, ct_master, loc, NULL);
818 #endif
819  }
820 
821  return status;
822 }
823 
832 void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid) {
833  KC_TRACE(10, ("__kmpc_end_master: called T#%d\n", global_tid));
834  __kmp_assert_valid_gtid(global_tid);
835  KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
836  KMP_POP_PARTITIONED_TIMER();
837 
838 #if OMPT_SUPPORT && OMPT_OPTIONAL
839  kmp_info_t *this_thr = __kmp_threads[global_tid];
840  kmp_team_t *team = this_thr->th.th_team;
841  if (ompt_enabled.ompt_callback_masked) {
842  int tid = __kmp_tid_from_gtid(global_tid);
843  ompt_callbacks.ompt_callback(ompt_callback_masked)(
844  ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
845  &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
846  OMPT_GET_RETURN_ADDRESS(0));
847  }
848 #endif
849 
850  if (__kmp_env_consistency_check) {
851  if (KMP_MASTER_GTID(global_tid))
852  __kmp_pop_sync(global_tid, ct_master, loc);
853  }
854 }
855 
864 kmp_int32 __kmpc_masked(ident_t *loc, kmp_int32 global_tid, kmp_int32 filter) {
865  int status = 0;
866  int tid;
867  KC_TRACE(10, ("__kmpc_masked: called T#%d\n", global_tid));
868  __kmp_assert_valid_gtid(global_tid);
869 
870  if (!TCR_4(__kmp_init_parallel))
871  __kmp_parallel_initialize();
872 
873  __kmp_resume_if_soft_paused();
874 
875  tid = __kmp_tid_from_gtid(global_tid);
876  if (tid == filter) {
877  KMP_COUNT_BLOCK(OMP_MASKED);
878  KMP_PUSH_PARTITIONED_TIMER(OMP_masked);
879  status = 1;
880  }
881 
882 #if OMPT_SUPPORT && OMPT_OPTIONAL
883  if (status) {
884  if (ompt_enabled.ompt_callback_masked) {
885  kmp_info_t *this_thr = __kmp_threads[global_tid];
886  kmp_team_t *team = this_thr->th.th_team;
887  ompt_callbacks.ompt_callback(ompt_callback_masked)(
888  ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
889  &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
890  OMPT_GET_RETURN_ADDRESS(0));
891  }
892  }
893 #endif
894 
895  if (__kmp_env_consistency_check) {
896 #if KMP_USE_DYNAMIC_LOCK
897  if (status)
898  __kmp_push_sync(global_tid, ct_masked, loc, NULL, 0);
899  else
900  __kmp_check_sync(global_tid, ct_masked, loc, NULL, 0);
901 #else
902  if (status)
903  __kmp_push_sync(global_tid, ct_masked, loc, NULL);
904  else
905  __kmp_check_sync(global_tid, ct_masked, loc, NULL);
906 #endif
907  }
908 
909  return status;
910 }
911 
920 void __kmpc_end_masked(ident_t *loc, kmp_int32 global_tid) {
921  KC_TRACE(10, ("__kmpc_end_masked: called T#%d\n", global_tid));
922  __kmp_assert_valid_gtid(global_tid);
923  KMP_POP_PARTITIONED_TIMER();
924 
925 #if OMPT_SUPPORT && OMPT_OPTIONAL
926  kmp_info_t *this_thr = __kmp_threads[global_tid];
927  kmp_team_t *team = this_thr->th.th_team;
928  if (ompt_enabled.ompt_callback_masked) {
929  int tid = __kmp_tid_from_gtid(global_tid);
930  ompt_callbacks.ompt_callback(ompt_callback_masked)(
931  ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
932  &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
933  OMPT_GET_RETURN_ADDRESS(0));
934  }
935 #endif
936 
937  if (__kmp_env_consistency_check) {
938  __kmp_pop_sync(global_tid, ct_masked, loc);
939  }
940 }
941 
949 void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
950  int cid = 0;
951  kmp_info_t *th;
952  KMP_DEBUG_ASSERT(__kmp_init_serial);
953 
954  KC_TRACE(10, ("__kmpc_ordered: called T#%d\n", gtid));
955  __kmp_assert_valid_gtid(gtid);
956 
957  if (!TCR_4(__kmp_init_parallel))
958  __kmp_parallel_initialize();
959 
960  __kmp_resume_if_soft_paused();
961 
962 #if USE_ITT_BUILD
963  __kmp_itt_ordered_prep(gtid);
964 // TODO: ordered_wait_id
965 #endif /* USE_ITT_BUILD */
966 
967  th = __kmp_threads[gtid];
968 
969 #if OMPT_SUPPORT && OMPT_OPTIONAL
970  kmp_team_t *team;
971  ompt_wait_id_t lck;
972  void *codeptr_ra;
973  OMPT_STORE_RETURN_ADDRESS(gtid);
974  if (ompt_enabled.enabled) {
975  team = __kmp_team_from_gtid(gtid);
976  lck = (ompt_wait_id_t)(uintptr_t)&team->t.t_ordered.dt.t_value;
977  /* OMPT state update */
978  th->th.ompt_thread_info.wait_id = lck;
979  th->th.ompt_thread_info.state = ompt_state_wait_ordered;
980 
981  /* OMPT event callback */
982  codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
983  if (ompt_enabled.ompt_callback_mutex_acquire) {
984  ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
985  ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin, lck,
986  codeptr_ra);
987  }
988  }
989 #endif
990 
991  if (th->th.th_dispatch->th_deo_fcn != 0)
992  (*th->th.th_dispatch->th_deo_fcn)(&gtid, &cid, loc);
993  else
994  __kmp_parallel_deo(&gtid, &cid, loc);
995 
996 #if OMPT_SUPPORT && OMPT_OPTIONAL
997  if (ompt_enabled.enabled) {
998  /* OMPT state update */
999  th->th.ompt_thread_info.state = ompt_state_work_parallel;
1000  th->th.ompt_thread_info.wait_id = 0;
1001 
1002  /* OMPT event callback */
1003  if (ompt_enabled.ompt_callback_mutex_acquired) {
1004  ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1005  ompt_mutex_ordered, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1006  }
1007  }
1008 #endif
1009 
1010 #if USE_ITT_BUILD
1011  __kmp_itt_ordered_start(gtid);
1012 #endif /* USE_ITT_BUILD */
1013 }
1014 
1022 void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid) {
1023  int cid = 0;
1024  kmp_info_t *th;
1025 
1026  KC_TRACE(10, ("__kmpc_end_ordered: called T#%d\n", gtid));
1027  __kmp_assert_valid_gtid(gtid);
1028 
1029 #if USE_ITT_BUILD
1030  __kmp_itt_ordered_end(gtid);
1031 // TODO: ordered_wait_id
1032 #endif /* USE_ITT_BUILD */
1033 
1034  th = __kmp_threads[gtid];
1035 
1036  if (th->th.th_dispatch->th_dxo_fcn != 0)
1037  (*th->th.th_dispatch->th_dxo_fcn)(&gtid, &cid, loc);
1038  else
1039  __kmp_parallel_dxo(&gtid, &cid, loc);
1040 
1041 #if OMPT_SUPPORT && OMPT_OPTIONAL
1042  OMPT_STORE_RETURN_ADDRESS(gtid);
1043  if (ompt_enabled.ompt_callback_mutex_released) {
1044  ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1045  ompt_mutex_ordered,
1046  (ompt_wait_id_t)(uintptr_t)&__kmp_team_from_gtid(gtid)
1047  ->t.t_ordered.dt.t_value,
1048  OMPT_LOAD_RETURN_ADDRESS(gtid));
1049  }
1050 #endif
1051 }
1052 
1053 #if KMP_USE_DYNAMIC_LOCK
1054 
1055 static __forceinline void
1056 __kmp_init_indirect_csptr(kmp_critical_name *crit, ident_t const *loc,
1057  kmp_int32 gtid, kmp_indirect_locktag_t tag) {
1058  // Pointer to the allocated indirect lock is written to crit, while indexing
1059  // is ignored.
1060  void *idx;
1061  kmp_indirect_lock_t **lck;
1062  lck = (kmp_indirect_lock_t **)crit;
1063  kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
1064  KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
1065  KMP_SET_I_LOCK_LOCATION(ilk, loc);
1066  KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
1067  KA_TRACE(20,
1068  ("__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
1069 #if USE_ITT_BUILD
1070  __kmp_itt_critical_creating(ilk->lock, loc);
1071 #endif
1072  int status = KMP_COMPARE_AND_STORE_PTR(lck, nullptr, ilk);
1073  if (status == 0) {
1074 #if USE_ITT_BUILD
1075  __kmp_itt_critical_destroyed(ilk->lock);
1076 #endif
1077  // We don't really need to destroy the unclaimed lock here since it will be
1078  // cleaned up at program exit.
1079  // KMP_D_LOCK_FUNC(&idx, destroy)((kmp_dyna_lock_t *)&idx);
1080  }
1081  KMP_DEBUG_ASSERT(*lck != NULL);
1082 }
1083 
1084 // Fast-path acquire tas lock
1085 #define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \
1086  { \
1087  kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1088  kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1089  kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1090  if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1091  !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \
1092  kmp_uint32 spins; \
1093  KMP_FSYNC_PREPARE(l); \
1094  KMP_INIT_YIELD(spins); \
1095  kmp_backoff_t backoff = __kmp_spin_backoff_params; \
1096  do { \
1097  if (TCR_4(__kmp_nth) > \
1098  (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
1099  KMP_YIELD(TRUE); \
1100  } else { \
1101  KMP_YIELD_SPIN(spins); \
1102  } \
1103  __kmp_spin_backoff(&backoff); \
1104  } while ( \
1105  KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1106  !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)); \
1107  } \
1108  KMP_FSYNC_ACQUIRED(l); \
1109  }
1110 
1111 // Fast-path test tas lock
1112 #define KMP_TEST_TAS_LOCK(lock, gtid, rc) \
1113  { \
1114  kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1115  kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1116  kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1117  rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \
1118  __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \
1119  }
1120 
1121 // Fast-path release tas lock
1122 #define KMP_RELEASE_TAS_LOCK(lock, gtid) \
1123  { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); }
1124 
1125 #if KMP_USE_FUTEX
1126 
1127 #include <sys/syscall.h>
1128 #include <unistd.h>
1129 #ifndef FUTEX_WAIT
1130 #define FUTEX_WAIT 0
1131 #endif
1132 #ifndef FUTEX_WAKE
1133 #define FUTEX_WAKE 1
1134 #endif
1135 
1136 // Fast-path acquire futex lock
1137 #define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \
1138  { \
1139  kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1140  kmp_int32 gtid_code = (gtid + 1) << 1; \
1141  KMP_MB(); \
1142  KMP_FSYNC_PREPARE(ftx); \
1143  kmp_int32 poll_val; \
1144  while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \
1145  &(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1146  KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \
1147  kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \
1148  if (!cond) { \
1149  if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \
1150  poll_val | \
1151  KMP_LOCK_BUSY(1, futex))) { \
1152  continue; \
1153  } \
1154  poll_val |= KMP_LOCK_BUSY(1, futex); \
1155  } \
1156  kmp_int32 rc; \
1157  if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \
1158  NULL, NULL, 0)) != 0) { \
1159  continue; \
1160  } \
1161  gtid_code |= 1; \
1162  } \
1163  KMP_FSYNC_ACQUIRED(ftx); \
1164  }
1165 
1166 // Fast-path test futex lock
1167 #define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \
1168  { \
1169  kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1170  if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1171  KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \
1172  KMP_FSYNC_ACQUIRED(ftx); \
1173  rc = TRUE; \
1174  } else { \
1175  rc = FALSE; \
1176  } \
1177  }
1178 
1179 // Fast-path release futex lock
1180 #define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \
1181  { \
1182  kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1183  KMP_MB(); \
1184  KMP_FSYNC_RELEASING(ftx); \
1185  kmp_int32 poll_val = \
1186  KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \
1187  if (KMP_LOCK_STRIP(poll_val) & 1) { \
1188  syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \
1189  KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \
1190  } \
1191  KMP_MB(); \
1192  KMP_YIELD_OVERSUB(); \
1193  }
1194 
1195 #endif // KMP_USE_FUTEX
1196 
1197 #else // KMP_USE_DYNAMIC_LOCK
1198 
1199 static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1200  ident_t const *loc,
1201  kmp_int32 gtid) {
1202  kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1203 
1204  // Because of the double-check, the following load doesn't need to be volatile
1205  kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1206 
1207  if (lck == NULL) {
1208  void *idx;
1209 
1210  // Allocate & initialize the lock.
1211  // Remember alloc'ed locks in table in order to free them in __kmp_cleanup()
1212  lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1213  __kmp_init_user_lock_with_checks(lck);
1214  __kmp_set_user_lock_location(lck, loc);
1215 #if USE_ITT_BUILD
1216  __kmp_itt_critical_creating(lck);
1217 // __kmp_itt_critical_creating() should be called *before* the first usage
1218 // of underlying lock. It is the only place where we can guarantee it. There
1219 // are chances the lock will destroyed with no usage, but it is not a
1220 // problem, because this is not real event seen by user but rather setting
1221 // name for object (lock). See more details in kmp_itt.h.
1222 #endif /* USE_ITT_BUILD */
1223 
1224  // Use a cmpxchg instruction to slam the start of the critical section with
1225  // the lock pointer. If another thread beat us to it, deallocate the lock,
1226  // and use the lock that the other thread allocated.
1227  int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1228 
1229  if (status == 0) {
1230 // Deallocate the lock and reload the value.
1231 #if USE_ITT_BUILD
1232  __kmp_itt_critical_destroyed(lck);
1233 // Let ITT know the lock is destroyed and the same memory location may be reused
1234 // for another purpose.
1235 #endif /* USE_ITT_BUILD */
1236  __kmp_destroy_user_lock_with_checks(lck);
1237  __kmp_user_lock_free(&idx, gtid, lck);
1238  lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1239  KMP_DEBUG_ASSERT(lck != NULL);
1240  }
1241  }
1242  return lck;
1243 }
1244 
1245 #endif // KMP_USE_DYNAMIC_LOCK
1246 
1257 void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1258  kmp_critical_name *crit) {
1259 #if KMP_USE_DYNAMIC_LOCK
1260 #if OMPT_SUPPORT && OMPT_OPTIONAL
1261  OMPT_STORE_RETURN_ADDRESS(global_tid);
1262 #endif // OMPT_SUPPORT
1263  __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1264 #else
1265  KMP_COUNT_BLOCK(OMP_CRITICAL);
1266 #if OMPT_SUPPORT && OMPT_OPTIONAL
1267  ompt_state_t prev_state = ompt_state_undefined;
1268  ompt_thread_info_t ti;
1269 #endif
1270  kmp_user_lock_p lck;
1271 
1272  KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
1273  __kmp_assert_valid_gtid(global_tid);
1274 
1275  // TODO: add THR_OVHD_STATE
1276 
1277  KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1278  KMP_CHECK_USER_LOCK_INIT();
1279 
1280  if ((__kmp_user_lock_kind == lk_tas) &&
1281  (sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1282  lck = (kmp_user_lock_p)crit;
1283  }
1284 #if KMP_USE_FUTEX
1285  else if ((__kmp_user_lock_kind == lk_futex) &&
1286  (sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1287  lck = (kmp_user_lock_p)crit;
1288  }
1289 #endif
1290  else { // ticket, queuing or drdpa
1291  lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1292  }
1293 
1294  if (__kmp_env_consistency_check)
1295  __kmp_push_sync(global_tid, ct_critical, loc, lck);
1296 
1297  // since the critical directive binds to all threads, not just the current
1298  // team we have to check this even if we are in a serialized team.
1299  // also, even if we are the uber thread, we still have to conduct the lock,
1300  // as we have to contend with sibling threads.
1301 
1302 #if USE_ITT_BUILD
1303  __kmp_itt_critical_acquiring(lck);
1304 #endif /* USE_ITT_BUILD */
1305 #if OMPT_SUPPORT && OMPT_OPTIONAL
1306  OMPT_STORE_RETURN_ADDRESS(gtid);
1307  void *codeptr_ra = NULL;
1308  if (ompt_enabled.enabled) {
1309  ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1310  /* OMPT state update */
1311  prev_state = ti.state;
1312  ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
1313  ti.state = ompt_state_wait_critical;
1314 
1315  /* OMPT event callback */
1316  codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1317  if (ompt_enabled.ompt_callback_mutex_acquire) {
1318  ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1319  ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1320  (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1321  }
1322  }
1323 #endif
1324  // Value of 'crit' should be good for using as a critical_id of the critical
1325  // section directive.
1326  __kmp_acquire_user_lock_with_checks(lck, global_tid);
1327 
1328 #if USE_ITT_BUILD
1329  __kmp_itt_critical_acquired(lck);
1330 #endif /* USE_ITT_BUILD */
1331 #if OMPT_SUPPORT && OMPT_OPTIONAL
1332  if (ompt_enabled.enabled) {
1333  /* OMPT state update */
1334  ti.state = prev_state;
1335  ti.wait_id = 0;
1336 
1337  /* OMPT event callback */
1338  if (ompt_enabled.ompt_callback_mutex_acquired) {
1339  ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1340  ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);