diff options
| -rw-r--r-- | llvm/include/llvm/IR/Instructions.h | 81 | ||||
| -rw-r--r-- | llvm/include/llvm/Support/AtomicOrdering.h | 151 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/AtomicExpandPass.cpp | 45 |
3 files changed, 118 insertions, 159 deletions
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index 19ea3464a10..7ff7900ad51 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -25,7 +25,6 @@ #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/InstrTypes.h" -#include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/ErrorHandling.h" #include <iterator> @@ -37,11 +36,91 @@ class ConstantRange; class DataLayout; class LLVMContext; +/// C++ defines ordering as a lattice. LLVM supplements this with NotAtomic and +/// Unordered, which are both below the C++ orders. See docs/Atomics.rst for +/// details. +/// +/// not_atomic-->unordered-->relaxed-->release--------------->acq_rel-->seq_cst +/// \-->consume-->acquire--/ +enum class AtomicOrdering { + NotAtomic = 0, + Unordered = 1, + Monotonic = 2, // Equivalent to C++'s relaxed. + // Consume = 3, // Not specified yet. + Acquire = 4, + Release = 5, + AcquireRelease = 6, + SequentiallyConsistent = 7 +}; + +bool operator<(AtomicOrdering, AtomicOrdering) = delete; +bool operator>(AtomicOrdering, AtomicOrdering) = delete; +bool operator<=(AtomicOrdering, AtomicOrdering) = delete; +bool operator>=(AtomicOrdering, AtomicOrdering) = delete; + +/// String used by LLVM IR to represent atomic ordering. +static inline const char *toIRString(AtomicOrdering ao) { + static const char *names[8] = {"not_atomic", "unordered", "monotonic", + "consume", "acquire", "release", + "acq_rel", "seq_cst"}; + return names[(size_t)ao]; +} + +/// Returns true if ao is stronger than other as defined by the AtomicOrdering +/// lattice, which is based on C++'s definition. +static inline bool isStrongerThan(AtomicOrdering ao, AtomicOrdering other) { + static const bool lookup[8][8] = { + // NA UN RX CO AC RE AR SC + /* NotAtomic */ {0, 0, 0, 0, 0, 0, 0, 0}, + /* Unordered */ {1, 0, 0, 0, 0, 0, 0, 0}, + /* relaxed */ {1, 1, 0, 0, 0, 0, 0, 0}, + /* consume */ {1, 1, 1, 0, 0, 0, 0, 0}, + /* acquire */ {1, 1, 1, 1, 0, 0, 0, 0}, + /* release */ {1, 1, 1, 0, 0, 0, 0, 0}, + /* acq_rel */ {1, 1, 1, 1, 1, 1, 0, 0}, + /* seq_cst */ {1, 1, 1, 1, 1, 1, 1, 0}, + }; + return lookup[(size_t)ao][(size_t)other]; +} + +static inline bool isAtLeastOrStrongerThan(AtomicOrdering ao, + AtomicOrdering other) { + static const bool lookup[8][8] = { + // NA UN RX CO AC RE AR SC + /* NotAtomic */ {1, 0, 0, 0, 0, 0, 0, 0}, + /* Unordered */ {1, 1, 0, 0, 0, 0, 0, 0}, + /* relaxed */ {1, 1, 1, 0, 0, 0, 0, 0}, + /* consume */ {1, 1, 1, 1, 0, 0, 0, 0}, + /* acquire */ {1, 1, 1, 1, 1, 0, 0, 0}, + /* release */ {1, 1, 1, 0, 0, 1, 0, 0}, + /* acq_rel */ {1, 1, 1, 1, 1, 1, 1, 0}, + /* seq_cst */ {1, 1, 1, 1, 1, 1, 1, 1}, + }; + return lookup[(size_t)ao][(size_t)other]; +} + +static inline bool isStrongerThanUnordered(AtomicOrdering Ord) { + return isStrongerThan(Ord, AtomicOrdering::Unordered); +} + +static inline bool isStrongerThanMonotonic(AtomicOrdering Ord) { + return isStrongerThan(Ord, AtomicOrdering::Monotonic); +} + +static inline bool isAcquireOrStronger(AtomicOrdering Ord) { + return isAtLeastOrStrongerThan(Ord, AtomicOrdering::Acquire); +} + +static inline bool isReleaseOrStronger(AtomicOrdering Ord) { + return isAtLeastOrStrongerThan(Ord, AtomicOrdering::Release); +} + enum SynchronizationScope { SingleThread = 0, CrossThread = 1 }; + //===----------------------------------------------------------------------===// // AllocaInst Class //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/Support/AtomicOrdering.h b/llvm/include/llvm/Support/AtomicOrdering.h deleted file mode 100644 index e0b828affa7..00000000000 --- a/llvm/include/llvm/Support/AtomicOrdering.h +++ /dev/null @@ -1,151 +0,0 @@ -//===-- llvm/Support/AtomicOrdering.h ---Atomic Ordering---------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// -/// \file -/// \brief Atomic ordering constants. -/// -/// These values are used by LLVM to represent atomic ordering for C++11's -/// memory model and more, as detailed in docs/Atomics.rst. -/// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_SUPPORT_ATOMICORDERING_H -#define LLVM_SUPPORT_ATOMICORDERING_H - -namespace llvm { - -/// Atomic ordering for C11 / C++11's memody models. -/// -/// These values cannot change because they are shared with standard library -/// implementations as well as with other compilers. -enum class AtomicOrderingCABI { - relaxed = 0, - consume = 1, - acquire = 2, - release = 3, - acq_rel = 4, - seq_cst = 5, -}; - -bool operator<(AtomicOrderingCABI, AtomicOrderingCABI) = delete; -bool operator>(AtomicOrderingCABI, AtomicOrderingCABI) = delete; -bool operator<=(AtomicOrderingCABI, AtomicOrderingCABI) = delete; -bool operator>=(AtomicOrderingCABI, AtomicOrderingCABI) = delete; - -// Validate an integral value which isn't known to fit within the enum's range -// is a valid AtomicOrderingCABI. -template <typename Int> static inline bool isValidAtomicOrderingCABI(Int I) { - return (Int)AtomicOrderingCABI::relaxed <= I && - I <= (Int)AtomicOrderingCABI::seq_cst; -} - -/// Atomic ordering for LLVM's memory model. -/// -/// C++ defines ordering as a lattice. LLVM supplements this with NotAtomic and -/// Unordered, which are both below the C++ orders. -/// -/// not_atomic-->unordered-->relaxed-->release--------------->acq_rel-->seq_cst -/// \-->consume-->acquire--/ -enum class AtomicOrdering { - NotAtomic = 0, - Unordered = 1, - Monotonic = 2, // Equivalent to C++'s relaxed. - // Consume = 3, // Not specified yet. - Acquire = 4, - Release = 5, - AcquireRelease = 6, - SequentiallyConsistent = 7 -}; - -bool operator<(AtomicOrdering, AtomicOrdering) = delete; -bool operator>(AtomicOrdering, AtomicOrdering) = delete; -bool operator<=(AtomicOrdering, AtomicOrdering) = delete; -bool operator>=(AtomicOrdering, AtomicOrdering) = delete; - -// Validate an integral value which isn't known to fit within the enum's range -// is a valid AtomicOrdering. -template <typename Int> static inline bool isValidAtomicOrdering(Int I) { - return (Int)AtomicOrdering::NotAtomic <= I && - I <= (Int)AtomicOrdering::SequentiallyConsistent; -} - -/// String used by LLVM IR to represent atomic ordering. -static inline const char *toIRString(AtomicOrdering ao) { - static const char *names[8] = {"not_atomic", "unordered", "monotonic", - "consume", "acquire", "release", - "acq_rel", "seq_cst"}; - return names[(size_t)ao]; -} - -/// Returns true if ao is stronger than other as defined by the AtomicOrdering -/// lattice, which is based on C++'s definition. -static inline bool isStrongerThan(AtomicOrdering ao, AtomicOrdering other) { - static const bool lookup[8][8] = { - // NA UN RX CO AC RE AR SC - /* NotAtomic */ {0, 0, 0, 0, 0, 0, 0, 0}, - /* Unordered */ {1, 0, 0, 0, 0, 0, 0, 0}, - /* relaxed */ {1, 1, 0, 0, 0, 0, 0, 0}, - /* consume */ {1, 1, 1, 0, 0, 0, 0, 0}, - /* acquire */ {1, 1, 1, 1, 0, 0, 0, 0}, - /* release */ {1, 1, 1, 0, 0, 0, 0, 0}, - /* acq_rel */ {1, 1, 1, 1, 1, 1, 0, 0}, - /* seq_cst */ {1, 1, 1, 1, 1, 1, 1, 0}, - }; - return lookup[(size_t)ao][(size_t)other]; -} - -static inline bool isAtLeastOrStrongerThan(AtomicOrdering ao, - AtomicOrdering other) { - static const bool lookup[8][8] = { - // NA UN RX CO AC RE AR SC - /* NotAtomic */ {1, 0, 0, 0, 0, 0, 0, 0}, - /* Unordered */ {1, 1, 0, 0, 0, 0, 0, 0}, - /* relaxed */ {1, 1, 1, 0, 0, 0, 0, 0}, - /* consume */ {1, 1, 1, 1, 0, 0, 0, 0}, - /* acquire */ {1, 1, 1, 1, 1, 0, 0, 0}, - /* release */ {1, 1, 1, 0, 0, 1, 0, 0}, - /* acq_rel */ {1, 1, 1, 1, 1, 1, 1, 0}, - /* seq_cst */ {1, 1, 1, 1, 1, 1, 1, 1}, - }; - return lookup[(size_t)ao][(size_t)other]; -} - -static inline bool isStrongerThanUnordered(AtomicOrdering ao) { - return isStrongerThan(ao, AtomicOrdering::Unordered); -} - -static inline bool isStrongerThanMonotonic(AtomicOrdering ao) { - return isStrongerThan(ao, AtomicOrdering::Monotonic); -} - -static inline bool isAcquireOrStronger(AtomicOrdering ao) { - return isAtLeastOrStrongerThan(ao, AtomicOrdering::Acquire); -} - -static inline bool isReleaseOrStronger(AtomicOrdering ao) { - return isAtLeastOrStrongerThan(ao, AtomicOrdering::Release); -} - -static inline AtomicOrderingCABI toCABI(AtomicOrdering ao) { - static const AtomicOrderingCABI lookup[8] = { - /* NotAtomic */ AtomicOrderingCABI::relaxed, - /* Unordered */ AtomicOrderingCABI::relaxed, - /* relaxed */ AtomicOrderingCABI::relaxed, - /* consume */ AtomicOrderingCABI::consume, - /* acquire */ AtomicOrderingCABI::acquire, - /* release */ AtomicOrderingCABI::release, - /* acq_rel */ AtomicOrderingCABI::acq_rel, - /* seq_cst */ AtomicOrderingCABI::seq_cst, - }; - return lookup[(size_t)ao]; -} - -} // End namespace llvm - -#endif diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp index 74cbea94542..e0c7fd6f883 100644 --- a/llvm/lib/CodeGen/AtomicExpandPass.cpp +++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp @@ -937,6 +937,39 @@ bool llvm::expandAtomicRMWToCmpXchg(AtomicRMWInst *AI, return true; } +// This converts from LLVM's internal AtomicOrdering enum to the +// memory_order_* value required by the __atomic_* libcalls. +static int libcallAtomicModel(AtomicOrdering AO) { + enum { + AO_ABI_memory_order_relaxed = 0, + AO_ABI_memory_order_consume = 1, + AO_ABI_memory_order_acquire = 2, + AO_ABI_memory_order_release = 3, + AO_ABI_memory_order_acq_rel = 4, + AO_ABI_memory_order_seq_cst = 5 + }; + + switch (AO) { + case AtomicOrdering::NotAtomic: + llvm_unreachable("Expected atomic memory order."); + case AtomicOrdering::Unordered: + case AtomicOrdering::Monotonic: + return AO_ABI_memory_order_relaxed; + // Not implemented yet in llvm: + // case AtomicOrdering::Consume: + // return AO_ABI_memory_order_consume; + case AtomicOrdering::Acquire: + return AO_ABI_memory_order_acquire; + case AtomicOrdering::Release: + return AO_ABI_memory_order_release; + case AtomicOrdering::AcquireRelease: + return AO_ABI_memory_order_acq_rel; + case AtomicOrdering::SequentiallyConsistent: + return AO_ABI_memory_order_seq_cst; + } + llvm_unreachable("Unknown atomic memory order."); +} + // In order to use one of the sized library calls such as // __atomic_fetch_add_4, the alignment must be sufficient, the size // must be one of the potentially-specialized sizes, and the value @@ -1118,14 +1151,12 @@ bool AtomicExpand::expandAtomicOpToLibcall( // TODO: the "order" argument type is "int", not int32. So // getInt32Ty may be wrong if the arch uses e.g. 16-bit ints. ConstantInt *SizeVal64 = ConstantInt::get(Type::getInt64Ty(Ctx), Size); - assert(Ordering != AtomicOrdering::NotAtomic && "expect atomic MO"); Constant *OrderingVal = - ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(Ordering)); - assert(Ordering2 != AtomicOrdering::NotAtomic && "expect atomic MO"); - Constant *Ordering2Val = - CASExpected - ? ConstantInt::get(Type::getInt32Ty(Ctx), (int)toCABI(Ordering2)) - : nullptr; + ConstantInt::get(Type::getInt32Ty(Ctx), libcallAtomicModel(Ordering)); + Constant *Ordering2Val = CASExpected + ? ConstantInt::get(Type::getInt32Ty(Ctx), + libcallAtomicModel(Ordering2)) + : nullptr; bool HasResult = I->getType() != Type::getVoidTy(Ctx); RTLIB::Libcall RTLibType; |
