LLVM OpenMP* Runtime Library
kmp_atomic.cpp
1 /*
2  * kmp_atomic.cpp -- ATOMIC implementation routines
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // The LLVM Compiler Infrastructure
8 //
9 // This file is dual licensed under the MIT and the University of Illinois Open
10 // Source Licenses. See LICENSE.txt for details.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "kmp_atomic.h"
15 #include "kmp.h" // TRUE, asm routines prototypes
16 
17 typedef unsigned char uchar;
18 typedef unsigned short ushort;
19 
562 /*
563  * Global vars
564  */
565 
566 #ifndef KMP_GOMP_COMPAT
567 int __kmp_atomic_mode = 1; // Intel perf
568 #else
569 int __kmp_atomic_mode = 2; // GOMP compatibility
570 #endif /* KMP_GOMP_COMPAT */
571 
572 KMP_ALIGN(128)
573 
574 // Control access to all user coded atomics in Gnu compat mode
575 kmp_atomic_lock_t __kmp_atomic_lock;
576 // Control access to all user coded atomics for 1-byte fixed data types
577 kmp_atomic_lock_t __kmp_atomic_lock_1i;
578 // Control access to all user coded atomics for 2-byte fixed data types
579 kmp_atomic_lock_t __kmp_atomic_lock_2i;
580 // Control access to all user coded atomics for 4-byte fixed data types
581 kmp_atomic_lock_t __kmp_atomic_lock_4i;
582 // Control access to all user coded atomics for kmp_real32 data type
583 kmp_atomic_lock_t __kmp_atomic_lock_4r;
584 // Control access to all user coded atomics for 8-byte fixed data types
585 kmp_atomic_lock_t __kmp_atomic_lock_8i;
586 // Control access to all user coded atomics for kmp_real64 data type
587 kmp_atomic_lock_t __kmp_atomic_lock_8r;
588 // Control access to all user coded atomics for complex byte data type
589 kmp_atomic_lock_t __kmp_atomic_lock_8c;
590 // Control access to all user coded atomics for long double data type
591 kmp_atomic_lock_t __kmp_atomic_lock_10r;
592 // Control access to all user coded atomics for _Quad data type
593 kmp_atomic_lock_t __kmp_atomic_lock_16r;
594 // Control access to all user coded atomics for double complex data type
595 kmp_atomic_lock_t __kmp_atomic_lock_16c;
596 // Control access to all user coded atomics for long double complex type
597 kmp_atomic_lock_t __kmp_atomic_lock_20c;
598 // Control access to all user coded atomics for _Quad complex data type
599 kmp_atomic_lock_t __kmp_atomic_lock_32c;
600 
601 /* 2007-03-02:
602  Without "volatile" specifier in OP_CMPXCHG and MIN_MAX_CMPXCHG we have a bug
603  on *_32 and *_32e. This is just a temporary workaround for the problem. It
604  seems the right solution is writing OP_CMPXCHG and MIN_MAX_CMPXCHG routines
605  in assembler language. */
606 #define KMP_ATOMIC_VOLATILE volatile
607 
608 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD
609 
610 static inline void operator+=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
611  lhs.q += rhs.q;
612 }
613 static inline void operator-=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
614  lhs.q -= rhs.q;
615 }
616 static inline void operator*=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
617  lhs.q *= rhs.q;
618 }
619 static inline void operator/=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
620  lhs.q /= rhs.q;
621 }
622 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
623  return lhs.q < rhs.q;
624 }
625 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
626  return lhs.q > rhs.q;
627 }
628 
629 static inline void operator+=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
630  lhs.q += rhs.q;
631 }
632 static inline void operator-=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
633  lhs.q -= rhs.q;
634 }
635 static inline void operator*=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
636  lhs.q *= rhs.q;
637 }
638 static inline void operator/=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
639  lhs.q /= rhs.q;
640 }
641 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
642  return lhs.q < rhs.q;
643 }
644 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
645  return lhs.q > rhs.q;
646 }
647 
648 static inline void operator+=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
649  lhs.q += rhs.q;
650 }
651 static inline void operator-=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
652  lhs.q -= rhs.q;
653 }
654 static inline void operator*=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
655  lhs.q *= rhs.q;
656 }
657 static inline void operator/=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
658  lhs.q /= rhs.q;
659 }
660 
661 static inline void operator+=(kmp_cmplx128_a16_t &lhs,
662  kmp_cmplx128_a16_t &rhs) {
663  lhs.q += rhs.q;
664 }
665 static inline void operator-=(kmp_cmplx128_a16_t &lhs,
666  kmp_cmplx128_a16_t &rhs) {
667  lhs.q -= rhs.q;
668 }
669 static inline void operator*=(kmp_cmplx128_a16_t &lhs,
670  kmp_cmplx128_a16_t &rhs) {
671  lhs.q *= rhs.q;
672 }
673 static inline void operator/=(kmp_cmplx128_a16_t &lhs,
674  kmp_cmplx128_a16_t &rhs) {
675  lhs.q /= rhs.q;
676 }
677 
678 #endif
679 
680 // ATOMIC implementation routines -----------------------------------------
681 // One routine for each operation and operand type.
682 // All routines declarations looks like
683 // void __kmpc_atomic_RTYPE_OP( ident_t*, int, TYPE *lhs, TYPE rhs );
684 
685 #define KMP_CHECK_GTID \
686  if (gtid == KMP_GTID_UNKNOWN) { \
687  gtid = __kmp_entry_gtid(); \
688  } // check and get gtid when needed
689 
690 // Beginning of a definition (provides name, parameters, gebug trace)
691 // TYPE_ID - operands type and size (fixed*, fixed*u for signed, unsigned
692 // fixed)
693 // OP_ID - operation identifier (add, sub, mul, ...)
694 // TYPE - operands' type
695 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
696  RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
697  TYPE *lhs, TYPE rhs) { \
698  KMP_DEBUG_ASSERT(__kmp_init_serial); \
699  KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
700 
701 // ------------------------------------------------------------------------
702 // Lock variables used for critical sections for various size operands
703 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
704 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
705 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
706 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
707 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
708 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
709 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
710 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
711 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
712 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
713 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
714 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
715 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
716 
717 // ------------------------------------------------------------------------
718 // Operation on *lhs, rhs bound by critical section
719 // OP - operator (it's supposed to contain an assignment)
720 // LCK_ID - lock identifier
721 // Note: don't check gtid as it should always be valid
722 // 1, 2-byte - expect valid parameter, other - check before this macro
723 #define OP_CRITICAL(OP, LCK_ID) \
724  __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
725  \
726  (*lhs) OP(rhs); \
727  \
728  __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
729 
730 // ------------------------------------------------------------------------
731 // For GNU compatibility, we may need to use a critical section,
732 // even though it is not required by the ISA.
733 //
734 // On IA-32 architecture, all atomic operations except for fixed 4 byte add,
735 // sub, and bitwise logical ops, and 1 & 2 byte logical ops use a common
736 // critical section. On Intel(R) 64, all atomic operations are done with fetch
737 // and add or compare and exchange. Therefore, the FLAG parameter to this
738 // macro is either KMP_ARCH_X86 or 0 (or 1, for Intel-specific extension which
739 // require a critical section, where we predict that they will be implemented
740 // in the Gnu codegen by calling GOMP_atomic_start() / GOMP_atomic_end()).
741 //
742 // When the OP_GOMP_CRITICAL macro is used in a *CRITICAL* macro construct,
743 // the FLAG parameter should always be 1. If we know that we will be using
744 // a critical section, then we want to make certain that we use the generic
745 // lock __kmp_atomic_lock to protect the atomic update, and not of of the
746 // locks that are specialized based upon the size or type of the data.
747 //
748 // If FLAG is 0, then we are relying on dead code elimination by the build
749 // compiler to get rid of the useless block of code, and save a needless
750 // branch at runtime.
751 
752 #ifdef KMP_GOMP_COMPAT
753 #define OP_GOMP_CRITICAL(OP, FLAG) \
754  if ((FLAG) && (__kmp_atomic_mode == 2)) { \
755  KMP_CHECK_GTID; \
756  OP_CRITICAL(OP, 0); \
757  return; \
758  }
759 #else
760 #define OP_GOMP_CRITICAL(OP, FLAG)
761 #endif /* KMP_GOMP_COMPAT */
762 
763 #if KMP_MIC
764 #define KMP_DO_PAUSE _mm_delay_32(1)
765 #else
766 #define KMP_DO_PAUSE KMP_CPU_PAUSE()
767 #endif /* KMP_MIC */
768 
769 // ------------------------------------------------------------------------
770 // Operation on *lhs, rhs using "compare_and_store" routine
771 // TYPE - operands' type
772 // BITS - size in bits, used to distinguish low level calls
773 // OP - operator
774 #define OP_CMPXCHG(TYPE, BITS, OP) \
775  { \
776  TYPE old_value, new_value; \
777  old_value = *(TYPE volatile *)lhs; \
778  new_value = old_value OP rhs; \
779  while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
780  (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
781  *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
782  KMP_DO_PAUSE; \
783  \
784  old_value = *(TYPE volatile *)lhs; \
785  new_value = old_value OP rhs; \
786  } \
787  }
788 
789 #if USE_CMPXCHG_FIX
790 // 2007-06-25:
791 // workaround for C78287 (complex(kind=4) data type). lin_32, lin_32e, win_32
792 // and win_32e are affected (I verified the asm). Compiler ignores the volatile
793 // qualifier of the temp_val in the OP_CMPXCHG macro. This is a problem of the
794 // compiler. Related tracker is C76005, targeted to 11.0. I verified the asm of
795 // the workaround.
796 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
797  { \
798  struct _sss { \
799  TYPE cmp; \
800  kmp_int##BITS *vvv; \
801  }; \
802  struct _sss old_value, new_value; \
803  old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
804  new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
805  *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
806  new_value.cmp = old_value.cmp OP rhs; \
807  while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
808  (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
809  *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
810  KMP_DO_PAUSE; \
811  \
812  *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
813  new_value.cmp = old_value.cmp OP rhs; \
814  } \
815  }
816 // end of the first part of the workaround for C78287
817 #endif // USE_CMPXCHG_FIX
818 
819 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
820 
821 // ------------------------------------------------------------------------
822 // X86 or X86_64: no alignment problems ====================================
823 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
824  GOMP_FLAG) \
825  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
826  OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
827  /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
828  KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
829  }
830 // -------------------------------------------------------------------------
831 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
832  GOMP_FLAG) \
833  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
834  OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
835  OP_CMPXCHG(TYPE, BITS, OP) \
836  }
837 #if USE_CMPXCHG_FIX
838 // -------------------------------------------------------------------------
839 // workaround for C78287 (complex(kind=4) data type)
840 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
841  MASK, GOMP_FLAG) \
842  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
843  OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
844  OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
845  }
846 // end of the second part of the workaround for C78287
847 #endif
848 
849 #else
850 // -------------------------------------------------------------------------
851 // Code for other architectures that don't handle unaligned accesses.
852 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
853  GOMP_FLAG) \
854  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
855  OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
856  if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
857  /* OP used as a sign for subtraction: (lhs-rhs) --> (lhs+-rhs) */ \
858  KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
859  } else { \
860  KMP_CHECK_GTID; \
861  OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \
862  } \
863  }
864 // -------------------------------------------------------------------------
865 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
866  GOMP_FLAG) \
867  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
868  OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
869  if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
870  OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
871  } else { \
872  KMP_CHECK_GTID; \
873  OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \
874  } \
875  }
876 #if USE_CMPXCHG_FIX
877 // -------------------------------------------------------------------------
878 // workaround for C78287 (complex(kind=4) data type)
879 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
880  MASK, GOMP_FLAG) \
881  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
882  OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
883  if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
884  OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
885  } else { \
886  KMP_CHECK_GTID; \
887  OP_CRITICAL(OP## =, LCK_ID) /* unaligned address - use critical */ \
888  } \
889  }
890 // end of the second part of the workaround for C78287
891 #endif // USE_CMPXCHG_FIX
892 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
893 
894 // Routines for ATOMIC 4-byte operands addition and subtraction
895 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
896  0) // __kmpc_atomic_fixed4_add
897 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
898  0) // __kmpc_atomic_fixed4_sub
899 
900 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
901  KMP_ARCH_X86) // __kmpc_atomic_float4_add
902 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
903  KMP_ARCH_X86) // __kmpc_atomic_float4_sub
904 
905 // Routines for ATOMIC 8-byte operands addition and subtraction
906 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
907  KMP_ARCH_X86) // __kmpc_atomic_fixed8_add
908 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
909  KMP_ARCH_X86) // __kmpc_atomic_fixed8_sub
910 
911 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
912  KMP_ARCH_X86) // __kmpc_atomic_float8_add
913 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
914  KMP_ARCH_X86) // __kmpc_atomic_float8_sub
915 
916 // ------------------------------------------------------------------------
917 // Entries definition for integer operands
918 // TYPE_ID - operands type and size (fixed4, float4)
919 // OP_ID - operation identifier (add, sub, mul, ...)
920 // TYPE - operand type
921 // BITS - size in bits, used to distinguish low level calls
922 // OP - operator (used in critical section)
923 // LCK_ID - lock identifier, used to possibly distinguish lock variable
924 // MASK - used for alignment check
925 
926 // TYPE_ID,OP_ID, TYPE, BITS,OP,LCK_ID,MASK,GOMP_FLAG
927 // ------------------------------------------------------------------------
928 // Routines for ATOMIC integer operands, other operators
929 // ------------------------------------------------------------------------
930 // TYPE_ID,OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG
931 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
932  KMP_ARCH_X86) // __kmpc_atomic_fixed1_add
933 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
934  0) // __kmpc_atomic_fixed1_andb
935 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
936  KMP_ARCH_X86) // __kmpc_atomic_fixed1_div
937 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
938  KMP_ARCH_X86) // __kmpc_atomic_fixed1u_div
939 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
940  KMP_ARCH_X86) // __kmpc_atomic_fixed1_mul
941 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
942  0) // __kmpc_atomic_fixed1_orb
943 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
944  KMP_ARCH_X86) // __kmpc_atomic_fixed1_shl
945 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
946  KMP_ARCH_X86) // __kmpc_atomic_fixed1_shr
947 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
948  KMP_ARCH_X86) // __kmpc_atomic_fixed1u_shr
949 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
950  KMP_ARCH_X86) // __kmpc_atomic_fixed1_sub
951 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
952  0) // __kmpc_atomic_fixed1_xor
953 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
954  KMP_ARCH_X86) // __kmpc_atomic_fixed2_add
955 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
956  0) // __kmpc_atomic_fixed2_andb
957 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
958  KMP_ARCH_X86) // __kmpc_atomic_fixed2_div
959 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
960  KMP_ARCH_X86) // __kmpc_atomic_fixed2u_div
961 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
962  KMP_ARCH_X86) // __kmpc_atomic_fixed2_mul
963 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
964  0) // __kmpc_atomic_fixed2_orb
965 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
966  KMP_ARCH_X86) // __kmpc_atomic_fixed2_shl
967 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
968  KMP_ARCH_X86) // __kmpc_atomic_fixed2_shr
969 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
970  KMP_ARCH_X86) // __kmpc_atomic_fixed2u_shr
971 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
972  KMP_ARCH_X86) // __kmpc_atomic_fixed2_sub
973 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
974  0) // __kmpc_atomic_fixed2_xor
975 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
976  0) // __kmpc_atomic_fixed4_andb
977 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
978  KMP_ARCH_X86) // __kmpc_atomic_fixed4_div
979 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
980  KMP_ARCH_X86) // __kmpc_atomic_fixed4u_div
981 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
982  KMP_ARCH_X86) // __kmpc_atomic_fixed4_mul
983 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
984  0) // __kmpc_atomic_fixed4_orb
985 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
986  KMP_ARCH_X86) // __kmpc_atomic_fixed4_shl
987 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
988  KMP_ARCH_X86) // __kmpc_atomic_fixed4_shr
989 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
990  KMP_ARCH_X86) // __kmpc_atomic_fixed4u_shr
991 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
992  0) // __kmpc_atomic_fixed4_xor
993 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
994  KMP_ARCH_X86) // __kmpc_atomic_fixed8_andb
995 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
996  KMP_ARCH_X86) // __kmpc_atomic_fixed8_div
997 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
998  KMP_ARCH_X86) // __kmpc_atomic_fixed8u_div
999 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1000  KMP_ARCH_X86) // __kmpc_atomic_fixed8_mul
1001 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1002  KMP_ARCH_X86) // __kmpc_atomic_fixed8_orb
1003 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1004  KMP_ARCH_X86) // __kmpc_atomic_fixed8_shl
1005 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1006  KMP_ARCH_X86) // __kmpc_atomic_fixed8_shr
1007 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1008  KMP_ARCH_X86) // __kmpc_atomic_fixed8u_shr
1009 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1010  KMP_ARCH_X86) // __kmpc_atomic_fixed8_xor
1011 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1012  KMP_ARCH_X86) // __kmpc_atomic_float4_div
1013 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1014  KMP_ARCH_X86) // __kmpc_atomic_float4_mul
1015 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1016  KMP_ARCH_X86) // __kmpc_atomic_float8_div
1017 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1018  KMP_ARCH_X86) // __kmpc_atomic_float8_mul
1019 // TYPE_ID,OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG
1020 
1021 /* ------------------------------------------------------------------------ */
1022 /* Routines for C/C++ Reduction operators && and || */
1023 
1024 // ------------------------------------------------------------------------
1025 // Need separate macros for &&, || because there is no combined assignment
1026 // TODO: eliminate ATOMIC_CRIT_{L,EQV} macros as not used
1027 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1028  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1029  OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1030  OP_CRITICAL(= *lhs OP, LCK_ID) \
1031  }
1032 
1033 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1034 
1035 // ------------------------------------------------------------------------
1036 // X86 or X86_64: no alignment problems ===================================
1037 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1038  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1039  OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1040  OP_CMPXCHG(TYPE, BITS, OP) \
1041  }
1042 
1043 #else
1044 // ------------------------------------------------------------------------
1045 // Code for other architectures that don't handle unaligned accesses.
1046 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1047  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1048  OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1049  if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1050  OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
1051  } else { \
1052  KMP_CHECK_GTID; \
1053  OP_CRITICAL(= *lhs OP, LCK_ID) /* unaligned - use critical */ \
1054  } \
1055  }
1056 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1057 
1058 ATOMIC_CMPX_L(fixed1, andl, char, 8, &&, 1i, 0,
1059  KMP_ARCH_X86) // __kmpc_atomic_fixed1_andl
1060 ATOMIC_CMPX_L(fixed1, orl, char, 8, ||, 1i, 0,
1061  KMP_ARCH_X86) // __kmpc_atomic_fixed1_orl
1062 ATOMIC_CMPX_L(fixed2, andl, short, 16, &&, 2i, 1,
1063  KMP_ARCH_X86) // __kmpc_atomic_fixed2_andl
1064 ATOMIC_CMPX_L(fixed2, orl, short, 16, ||, 2i, 1,
1065  KMP_ARCH_X86) // __kmpc_atomic_fixed2_orl
1066 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1067  0) // __kmpc_atomic_fixed4_andl
1068 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1069  0) // __kmpc_atomic_fixed4_orl
1070 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1071  KMP_ARCH_X86) // __kmpc_atomic_fixed8_andl
1072 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1073  KMP_ARCH_X86) // __kmpc_atomic_fixed8_orl
1074 
1075 /* ------------------------------------------------------------------------- */
1076 /* Routines for Fortran operators that matched no one in C: */
1077 /* MAX, MIN, .EQV., .NEQV. */
1078 /* Operators .AND., .OR. are covered by __kmpc_atomic_*_{andl,orl} */
1079 /* Intrinsics IAND, IOR, IEOR are covered by __kmpc_atomic_*_{andb,orb,xor} */
1080 
1081 // -------------------------------------------------------------------------
1082 // MIN and MAX need separate macros
1083 // OP - operator to check if we need any actions?
1084 #define MIN_MAX_CRITSECT(OP, LCK_ID) \
1085  __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1086  \
1087  if (*lhs OP rhs) { /* still need actions? */ \
1088  *lhs = rhs; \
1089  } \
1090  __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1091 
1092 // -------------------------------------------------------------------------
1093 #ifdef KMP_GOMP_COMPAT
1094 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \
1095  if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1096  KMP_CHECK_GTID; \
1097  MIN_MAX_CRITSECT(OP, 0); \
1098  return; \
1099  }
1100 #else
1101 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG)
1102 #endif /* KMP_GOMP_COMPAT */
1103 
1104 // -------------------------------------------------------------------------
1105 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1106  { \
1107  TYPE KMP_ATOMIC_VOLATILE temp_val; \
1108  TYPE old_value; \
1109  temp_val = *lhs; \
1110  old_value = temp_val; \
1111  while (old_value OP rhs && /* still need actions? */ \
1112  !KMP_COMPARE_AND_STORE_ACQ##BITS( \
1113  (kmp_int##BITS *)lhs, \
1114  *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1115  *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
1116  KMP_CPU_PAUSE(); \
1117  temp_val = *lhs; \
1118  old_value = temp_val; \
1119  } \
1120  }
1121 
1122 // -------------------------------------------------------------------------
1123 // 1-byte, 2-byte operands - use critical section
1124 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1125  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1126  if (*lhs OP rhs) { /* need actions? */ \
1127  GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1128  MIN_MAX_CRITSECT(OP, LCK_ID) \
1129  } \
1130  }
1131 
1132 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1133 
1134 // -------------------------------------------------------------------------
1135 // X86 or X86_64: no alignment problems ====================================
1136 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1137  GOMP_FLAG) \
1138  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1139  if (*lhs OP rhs) { \
1140  GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1141  MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1142  } \
1143  }
1144 
1145 #else
1146 // -------------------------------------------------------------------------
1147 // Code for other architectures that don't handle unaligned accesses.
1148 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1149  GOMP_FLAG) \
1150  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1151  if (*lhs OP rhs) { \
1152  GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1153  if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1154  MIN_MAX_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
1155  } else { \
1156  KMP_CHECK_GTID; \
1157  MIN_MAX_CRITSECT(OP, LCK_ID) /* unaligned address */ \
1158  } \
1159  } \
1160  }
1161 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1162 
1163 MIN_MAX_COMPXCHG(fixed1, max, char, 8, <, 1i, 0,
1164  KMP_ARCH_X86) // __kmpc_atomic_fixed1_max
1165 MIN_MAX_COMPXCHG(fixed1, min, char, 8, >, 1i, 0,
1166  KMP_ARCH_X86) // __kmpc_atomic_fixed1_min
1167 MIN_MAX_COMPXCHG(fixed2, max, short, 16, <, 2i, 1,
1168  KMP_ARCH_X86) // __kmpc_atomic_fixed2_max
1169 MIN_MAX_COMPXCHG(fixed2, min, short, 16, >, 2i, 1,
1170  KMP_ARCH_X86) // __kmpc_atomic_fixed2_min
1171 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1172  0) // __kmpc_atomic_fixed4_max
1173 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1174  0) // __kmpc_atomic_fixed4_min
1175 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1176  KMP_ARCH_X86) // __kmpc_atomic_fixed8_max
1177 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1178  KMP_ARCH_X86) // __kmpc_atomic_fixed8_min
1179 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1180  KMP_ARCH_X86) // __kmpc_atomic_float4_max
1181 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1182  KMP_ARCH_X86) // __kmpc_atomic_float4_min
1183 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1184  KMP_ARCH_X86) // __kmpc_atomic_float8_max
1185 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1186  KMP_ARCH_X86) // __kmpc_atomic_float8_min
1187 #if KMP_HAVE_QUAD
1188 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1189  1) // __kmpc_atomic_float16_max
1190 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1191  1) // __kmpc_atomic_float16_min
1192 #if (KMP_ARCH_X86)
1193 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1194  1) // __kmpc_atomic_float16_max_a16
1195 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1196  1) // __kmpc_atomic_float16_min_a16
1197 #endif
1198 #endif
1199 // ------------------------------------------------------------------------
1200 // Need separate macros for .EQV. because of the need of complement (~)
1201 // OP ignored for critical sections, ^=~ used instead
1202 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1203  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1204  OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) /* send assignment */ \
1205  OP_CRITICAL(^= ~, LCK_ID) /* send assignment and complement */ \
1206  }
1207 
1208 // ------------------------------------------------------------------------
1209 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1210 // ------------------------------------------------------------------------
1211 // X86 or X86_64: no alignment problems ===================================
1212 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1213  GOMP_FLAG) \
1214  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1215  OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) /* send assignment */ \
1216  OP_CMPXCHG(TYPE, BITS, OP) \
1217  }
1218 // ------------------------------------------------------------------------
1219 #else
1220 // ------------------------------------------------------------------------
1221 // Code for other architectures that don't handle unaligned accesses.
1222 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1223  GOMP_FLAG) \
1224  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1225  OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \
1226  if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1227  OP_CMPXCHG(TYPE, BITS, OP) /* aligned address */ \
1228  } else { \
1229  KMP_CHECK_GTID; \
1230  OP_CRITICAL(^= ~, LCK_ID) /* unaligned address - use critical */ \
1231  } \
1232  }
1233 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1234 
1235 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1236  KMP_ARCH_X86) // __kmpc_atomic_fixed1_neqv
1237 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1238  KMP_ARCH_X86) // __kmpc_atomic_fixed2_neqv
1239 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1240  KMP_ARCH_X86) // __kmpc_atomic_fixed4_neqv
1241 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1242  KMP_ARCH_X86) // __kmpc_atomic_fixed8_neqv
1243 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1244  KMP_ARCH_X86) // __kmpc_atomic_fixed1_eqv
1245 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1246  KMP_ARCH_X86) // __kmpc_atomic_fixed2_eqv
1247 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1248  KMP_ARCH_X86) // __kmpc_atomic_fixed4_eqv
1249 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1250  KMP_ARCH_X86) // __kmpc_atomic_fixed8_eqv
1251 
1252 // ------------------------------------------------------------------------
1253 // Routines for Extended types: long double, _Quad, complex flavours (use
1254 // critical section)
1255 // TYPE_ID, OP_ID, TYPE - detailed above
1256 // OP - operator
1257 // LCK_ID - lock identifier, used to possibly distinguish lock variable
1258 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1259  ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1260  OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) /* send assignment */ \
1261  OP_CRITICAL(OP## =, LCK_ID) /* send assignment */ \
1262  }
1263 
1264 /* ------------------------------------------------------------------------- */
1265 // routines for long double type
1266 ATOMIC_CRITICAL(float10, add, long double, +, 10r,
1267  1) // __kmpc_atomic_float10_add
1268 ATOMIC_CRITICAL(float10, sub, long double, -, 10r,
1269  1) // __kmpc_atomic_float10_sub
1270 ATOMIC_CRITICAL(float10, mul, long double, *, 10r,
1271  1) // __kmpc_atomic_float10_mul
1272 ATOMIC_CRITICAL(float10, div, long double, /, 10r,
1273  1) // __kmpc_atomic_float10_div
1274 #if KMP_HAVE_QUAD
1275 // routines for _Quad type
1276 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1277  1) // __kmpc_atomic_float16_add
1278 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1279  1) // __kmpc_atomic_float16_sub
1280 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1281  1) // __kmpc_atomic_float16_mul
1282 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1283  1) // __kmpc_atomic_float16_div
1284 #if (KMP_ARCH_X86)
1285 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1286  1) // __kmpc_atomic_float16_add_a16
1287 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1288  1) // __kmpc_atomic_float16_sub_a16
1289 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1290  1) // __kmpc_atomic_float16_mul_a16
1291 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1292  1) // __kmpc_atomic_float16_div_a16
1293 #endif
1294 #endif
1295 // routines for complex types
1296 
1297 #if USE_CMPXCHG_FIX
1298 // workaround for C78287 (complex(kind=4) data type)
1299 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1300  1) // __kmpc_atomic_cmplx4_add
1301 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1302  1) // __kmpc_atomic_cmplx4_sub
1303 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1304  1) // __kmpc_atomic_cmplx4_mul
1305 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1306  1) // __kmpc_atomic_cmplx4_div
1307 // end of the workaround for C78287
1308 #else
1309 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1) // __kmpc_atomic_cmplx4_add
1310 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1) // __kmpc_atomic_cmplx4_sub
1311 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1) // __kmpc_atomic_cmplx4_mul
1312 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1) // __kmpc_atomic_cmplx4_div
1313 #endif // USE_CMPXCHG_FIX
1314 
1315 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1) // __kmpc_atomic_cmplx8_add
1316 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1) // __kmpc_atomic_cmplx8_sub
1317 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1) // __kmpc_atomic_cmplx8_mul
1318 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1) // __kmpc_atomic_cmplx8_div
1319 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1320  1) // __kmpc_atomic_cmplx10_add
<