summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp12
-rw-r--r--llvm/lib/IR/Function.cpp16
-rw-r--r--llvm/lib/IR/IRBuilder.cpp4
-rw-r--r--llvm/lib/IR/Verifier.cpp7
-rw-r--r--llvm/lib/Target/X86/X86TargetTransformInfo.cpp6
-rw-r--r--llvm/lib/Transforms/Vectorize/LoopVectorize.cpp20
6 files changed, 38 insertions, 27 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 81b824f305e..ae73e9f3560 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -3644,9 +3644,10 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
SDLoc sdl = getCurSDLoc();
- Value *PtrOperand = I.getArgOperand(0);
+ // llvm.masked.store.*(Src0, Ptr, alignemt, Mask)
+ Value *PtrOperand = I.getArgOperand(1);
SDValue Ptr = getValue(PtrOperand);
- SDValue Src0 = getValue(I.getArgOperand(1));
+ SDValue Src0 = getValue(I.getArgOperand(0));
SDValue Mask = getValue(I.getArgOperand(3));
EVT VT = Src0.getValueType();
unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
@@ -3669,14 +3670,15 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I) {
void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I) {
SDLoc sdl = getCurSDLoc();
+ // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
Value *PtrOperand = I.getArgOperand(0);
SDValue Ptr = getValue(PtrOperand);
- SDValue Src0 = getValue(I.getArgOperand(1));
- SDValue Mask = getValue(I.getArgOperand(3));
+ SDValue Src0 = getValue(I.getArgOperand(3));
+ SDValue Mask = getValue(I.getArgOperand(2));
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
EVT VT = TLI.getValueType(I.getType());
- unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
+ unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
if (!Alignment)
Alignment = DAG.getEVTAlignment(VT);
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index f462ec0890d..bfd0ca65a32 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -537,7 +537,8 @@ enum IIT_Info {
IIT_V1 = 27,
IIT_VARARG = 28,
IIT_HALF_VEC_ARG = 29,
- IIT_SAME_VEC_WIDTH_ARG = 30
+ IIT_SAME_VEC_WIDTH_ARG = 30,
+ IIT_PTR_TO_ARG = 31
};
@@ -651,6 +652,12 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
ArgInfo));
return;
}
+ case IIT_PTR_TO_ARG: {
+ unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+ OutputTable.push_back(IITDescriptor::get(IITDescriptor::PtrToArgument,
+ ArgInfo));
+ return;
+ }
case IIT_EMPTYSTRUCT:
OutputTable.push_back(IITDescriptor::get(IITDescriptor::Struct, 0));
return;
@@ -758,13 +765,18 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
case IITDescriptor::HalfVecArgument:
return VectorType::getHalfElementsVectorType(cast<VectorType>(
Tys[D.getArgumentNumber()]));
- case IITDescriptor::SameVecWidthArgument:
+ case IITDescriptor::SameVecWidthArgument: {
Type *EltTy = DecodeFixedType(Infos, Tys, Context);
Type *Ty = Tys[D.getArgumentNumber()];
if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
return VectorType::get(EltTy, VTy->getNumElements());
}
llvm_unreachable("unhandled");
+ }
+ case IITDescriptor::PtrToArgument: {
+ Type *Ty = Tys[D.getArgumentNumber()];
+ return PointerType::getUnqual(Ty);
+ }
}
llvm_unreachable("unhandled");
}
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 5f63ded48fe..31dc52e3fc2 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -187,7 +187,7 @@ CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
/// Ops - an array of operands.
CallInst *IRBuilderBase::CreateMaskedLoad(ArrayRef<Value *> Ops) {
// The only one overloaded type - the type of passthru value in this case
- Type *DataTy = Ops[1]->getType();
+ Type *DataTy = Ops[3]->getType();
return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops, DataTy);
}
@@ -195,7 +195,7 @@ CallInst *IRBuilderBase::CreateMaskedLoad(ArrayRef<Value *> Ops) {
/// Ops - an array of operands.
CallInst *IRBuilderBase::CreateMaskedStore(ArrayRef<Value *> Ops) {
// DataTy - type of the data to be stored - the only one overloaded type
- Type *DataTy = Ops[1]->getType();
+ Type *DataTy = Ops[0]->getType();
return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, DataTy);
}
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 66b6bb06018..0bc9d5e1bc8 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -2441,6 +2441,13 @@ bool Verifier::VerifyIntrinsicType(Type *Ty,
return VerifyIntrinsicType(ThisArgType->getVectorElementType(),
Infos, ArgTys);
}
+ case IITDescriptor::PtrToArgument: {
+ if (D.getArgumentNumber() >= ArgTys.size())
+ return true;
+ Type * ReferenceType = ArgTys[D.getArgumentNumber()];
+ PointerType *ThisArgType = dyn_cast<PointerType>(Ty);
+ return (!ThisArgType || ThisArgType->getElementType() != ReferenceType);
+ }
}
llvm_unreachable("unhandled");
}
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index c55f311b619..67488f7ad79 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1159,11 +1159,11 @@ unsigned X86TTI::getIntImmCost(Intrinsic::ID IID, unsigned Idx,
return X86TTI::getIntImmCost(Imm, Ty);
}
-bool X86TTI::isLegalMaskedLoad(Type *DataType, int Consecutive) const {
- int ScalarWidth = DataType->getScalarSizeInBits();
+bool X86TTI::isLegalMaskedLoad(Type *DataTy, int Consecutive) const {
+ int DataWidth = DataTy->getPrimitiveSizeInBits();
// Todo: AVX512 allows gather/scatter, works with strided and random as well
- if ((ScalarWidth < 32) || (Consecutive == 0))
+ if ((DataWidth < 32) || (Consecutive == 0))
return false;
if (ST->hasAVX512() || ST->hasAVX2())
return true;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d0860c44c0e..d0457287099 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1880,15 +1880,10 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
Instruction *NewSI;
if (Legal->isMaskRequired(SI)) {
- Type *I8PtrTy =
- Builder.getInt8PtrTy(PartPtr->getType()->getPointerAddressSpace());
-
- Value *I8Ptr = Builder.CreateBitCast(PartPtr, I8PtrTy);
-
VectorParts Cond = createBlockInMask(SI->getParent());
SmallVector <Value *, 8> Ops;
- Ops.push_back(I8Ptr);
Ops.push_back(StoredVal[Part]);
+ Ops.push_back(VecPtr);
Ops.push_back(Builder.getInt32(Alignment));
Ops.push_back(Cond[Part]);
NewSI = Builder.CreateMaskedStore(Ops);
@@ -1915,23 +1910,18 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) {
}
Instruction* NewLI;
+ Value *VecPtr = Builder.CreateBitCast(PartPtr,
+ DataTy->getPointerTo(AddressSpace));
if (Legal->isMaskRequired(LI)) {
- Type *I8PtrTy =
- Builder.getInt8PtrTy(PartPtr->getType()->getPointerAddressSpace());
-
- Value *I8Ptr = Builder.CreateBitCast(PartPtr, I8PtrTy);
-
VectorParts SrcMask = createBlockInMask(LI->getParent());
SmallVector <Value *, 8> Ops;
- Ops.push_back(I8Ptr);
- Ops.push_back(UndefValue::get(DataTy));
+ Ops.push_back(VecPtr);
Ops.push_back(Builder.getInt32(Alignment));
Ops.push_back(SrcMask[Part]);
+ Ops.push_back(UndefValue::get(DataTy));
NewLI = Builder.CreateMaskedLoad(Ops);
}
else {
- Value *VecPtr = Builder.CreateBitCast(PartPtr,
- DataTy->getPointerTo(AddressSpace));
NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load");
}
propagateMetadata(NewLI, LI);
OpenPOWER on IntegriCloud