summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/AsmParser/LLParser.cpp10
-rw-r--r--llvm/lib/CodeGen/AtomicExpandPass.cpp15
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp18
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp29
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h1
-rw-r--r--llvm/lib/CodeGen/TargetLoweringBase.cpp8
-rw-r--r--llvm/lib/IR/Verifier.cpp15
-rw-r--r--llvm/lib/Target/AArch64/AArch64ISelLowering.cpp14
8 files changed, 101 insertions, 9 deletions
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index ee634505581..816bb4e9401 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -6850,12 +6850,20 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
if (cast<PointerType>(Ptr->getType())->getElementType() != Val->getType())
return Error(ValLoc, "atomicrmw value and pointer type do not match");
- if (!Val->getType()->isIntegerTy()) {
+ if (Operation != AtomicRMWInst::Xchg && !Val->getType()->isIntegerTy()) {
return Error(ValLoc, "atomicrmw " +
AtomicRMWInst::getOperationName(Operation) +
" operand must be an integer");
}
+ if (Operation == AtomicRMWInst::Xchg &&
+ !Val->getType()->isIntegerTy() &&
+ !Val->getType()->isFloatingPointTy()) {
+ return Error(ValLoc, "atomicrmw " +
+ AtomicRMWInst::getOperationName(Operation) +
+ " operand must be an integer or floating point type");
+ }
+
unsigned Size = Val->getType()->getPrimitiveSizeInBits();
if (Size < 8 || (Size & (Size - 1)))
return Error(ValLoc, "atomicrmw operand must be power-of-two byte-sized"
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index 95581c09dd1..9163e76eb14 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -496,11 +496,26 @@ static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
Value *Loaded, Value *NewVal,
AtomicOrdering MemOpOrder,
Value *&Success, Value *&NewLoaded) {
+ Type *OrigTy = NewVal->getType();
+
+ // This code can go away when cmpxchg supports FP types.
+ bool NeedBitcast = OrigTy->isFloatingPointTy();
+ if (NeedBitcast) {
+ IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits());
+ unsigned AS = Addr->getType()->getPointerAddressSpace();
+ Addr = Builder.CreateBitCast(Addr, IntTy->getPointerTo(AS));
+ NewVal = Builder.CreateBitCast(NewVal, IntTy);
+ Loaded = Builder.CreateBitCast(Loaded, IntTy);
+ }
+
Value* Pair = Builder.CreateAtomicCmpXchg(
Addr, Loaded, NewVal, MemOpOrder,
AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
Success = Builder.CreateExtractValue(Pair, 1, "success");
NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
+
+ if (NeedBitcast)
+ NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
}
/// Emit IR to implement the given atomicrmw operation on values in registers,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index d3aea37f944..24ddfb9585c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4532,6 +4532,24 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Results.push_back(CvtVec);
break;
}
+ case ISD::ATOMIC_SWAP: {
+ AtomicSDNode *AM = cast<AtomicSDNode>(Node);
+ SDLoc SL(Node);
+ SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
+ assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
+ "unexpected promotion type");
+ assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
+ "unexpected atomic_swap with illegal type");
+
+ SDValue NewAtomic
+ = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
+ DAG.getVTList(NVT, MVT::Other),
+ { AM->getChain(), AM->getBasePtr(), CastVal },
+ AM->getMemOperand());
+ Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
+ Results.push_back(NewAtomic.getValue(1));
+ break;
+ }
}
// Replace the original node with the legalized result.
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 4644e9588e7..f08526fb0ef 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -104,6 +104,7 @@ bool DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
case ISD::LOAD: R = SoftenFloatRes_LOAD(N, ResNo); break;
+ case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
case ISD::SELECT: R = SoftenFloatRes_SELECT(N, ResNo); break;
case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N, ResNo); break;
case ISD::SINT_TO_FP:
@@ -1932,7 +1933,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
-
+ case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
}
if (R.getNode())
@@ -2166,3 +2167,29 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
N->getValueType(0)));
}
+SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
+ EVT VT = N->getValueType(0);
+ EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+
+ AtomicSDNode *AM = cast<AtomicSDNode>(N);
+ SDLoc SL(N);
+
+ SDValue CastVal = BitConvertToInteger(AM->getVal());
+ EVT CastVT = CastVal.getValueType();
+
+ SDValue NewAtomic
+ = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
+ DAG.getVTList(CastVT, MVT::Other),
+ { AM->getChain(), AM->getBasePtr(), CastVal },
+ AM->getMemOperand());
+
+ SDValue ResultCast = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
+ NewAtomic);
+ // Legalize the chain result by replacing uses of the old value chain with the
+ // new one
+ ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
+
+ return ResultCast;
+
+}
+
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 032000f6cb7..737d9bd9634 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -640,6 +640,7 @@ private:
SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
SDValue PromoteFloatRes_UnaryOp(SDNode *N);
SDValue PromoteFloatRes_UNDEF(SDNode *N);
+ SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index e8619037564..fcc09e2f8de 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -583,6 +583,14 @@ void TargetLoweringBase::initActions() {
std::fill(std::begin(TargetDAGCombineArray),
std::end(TargetDAGCombineArray), 0);
+ for (MVT VT : MVT::fp_valuetypes()) {
+ MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
+ if (IntVT.isValid()) {
+ setOperationAction(ISD::ATOMIC_SWAP, VT, Promote);
+ AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT);
+ }
+ }
+
// Set default actions for various operations.
for (MVT VT : MVT::all_valuetypes()) {
// Default all indexed load / store to expand.
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 30e77b92009..338fe931883 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3431,10 +3431,17 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
PointerType *PTy = dyn_cast<PointerType>(RMWI.getOperand(0)->getType());
Assert(PTy, "First atomicrmw operand must be a pointer.", &RMWI);
Type *ElTy = PTy->getElementType();
- Assert(ElTy->isIntegerTy(), "atomicrmw " +
- AtomicRMWInst::getOperationName(Op) +
- " operand must have integer type!",
- &RMWI, ElTy);
+ if (Op == AtomicRMWInst::Xchg) {
+ Assert(ElTy->isIntegerTy() || ElTy->isFloatingPointTy(), "atomicrmw " +
+ AtomicRMWInst::getOperationName(Op) +
+ " operand must have integer or floating point type!",
+ &RMWI, ElTy);
+ } else {
+ Assert(ElTy->isIntegerTy(), "atomicrmw " +
+ AtomicRMWInst::getOperationName(Op) +
+ " operand must have integer type!",
+ &RMWI, ElTy);
+ }
checkAtomicMemAccessSize(ElTy, &RMWI);
Assert(ElTy == RMWI.getOperand(1)->getType(),
"Argument value type does not match pointer operand type!", &RMWI,
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 762f4413d72..1e509eca88f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11655,9 +11655,13 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
- return Builder.CreateTruncOrBitCast(
- Builder.CreateCall(Ldxr, Addr),
- cast<PointerType>(Addr->getType())->getElementType());
+ Type *EltTy = cast<PointerType>(Addr->getType())->getElementType();
+
+ const DataLayout &DL = M->getDataLayout();
+ IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(EltTy));
+ Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr, Addr), IntEltTy);
+
+ return Builder.CreateBitCast(Trunc, EltTy);
}
void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
@@ -11692,6 +11696,10 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder,
Type *Tys[] = { Addr->getType() };
Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
+ const DataLayout &DL = M->getDataLayout();
+ IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
+ Val = Builder.CreateBitCast(Val, IntValTy);
+
return Builder.CreateCall(Stxr,
{Builder.CreateZExtOrBitCast(
Val, Stxr->getFunctionType()->getParamType(0)),
OpenPOWER on IntegriCloud