summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp4
-rw-r--r--llvm/lib/IR/AutoUpgrade.cpp70
-rw-r--r--llvm/lib/IR/IRBuilder.cpp36
-rw-r--r--llvm/lib/IR/Verifier.cpp19
-rw-r--r--llvm/lib/Target/AArch64/AArch64FastISel.cpp4
-rw-r--r--llvm/lib/Target/ARM/ARMFastISel.cpp4
-rw-r--r--llvm/lib/Target/Mips/MipsFastISel.cpp4
-rw-r--r--llvm/lib/Target/X86/X86FastISel.cpp4
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp1
-rw-r--r--llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp21
10 files changed, 127 insertions, 40 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 27af4296552..1295b83fc6c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5017,6 +5017,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment.
bool isVol = MCI.isVolatile();
bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
+ // FIXME: Support passing different dest/src alignments to the memcpy DAG
+ // node.
SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
false, isTC,
MachinePointerInfo(I.getArgOperand(0)),
@@ -5049,6 +5051,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment.
bool isVol = MMI.isVolatile();
bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
+ // FIXME: Support passing different dest/src alignments to the memmove DAG
+ // node.
SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
isTC, MachinePointerInfo(I.getArgOperand(0)),
MachinePointerInfo(I.getArgOperand(1)));
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index bf96e772394..5d219995aed 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -22,6 +22,7 @@
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
@@ -522,6 +523,37 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
return true;
}
}
+ // Updating the memory intrinsics (memcpy/memmove/memset) that have an
+ // alignment parameter to embedding the alignment as an attribute of
+ // the pointer args.
+ if (Name.startswith("memcpy.") && F->arg_size() == 5) {
+ rename(F);
+ // Get the types of dest, src, and len
+ ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
+ ParamTypes);
+ return true;
+ }
+ if (Name.startswith("memmove.") && F->arg_size() == 5) {
+ rename(F);
+ // Get the types of dest, src, and len
+ ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
+ ParamTypes);
+ return true;
+ }
+ if (Name.startswith("memset.") && F->arg_size() == 5) {
+ rename(F);
+ // Get the types of dest, and len
+ const auto *FT = F->getFunctionType();
+ Type *ParamTypes[2] = {
+ FT->getParamType(0), // Dest
+ FT->getParamType(2) // len
+ };
+ NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
+ ParamTypes);
+ return true;
+ }
break;
}
case 'n': {
@@ -2198,14 +2230,17 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
return;
}
- CallInst *NewCall = nullptr;
- switch (NewFn->getIntrinsicID()) {
- default: {
+ const auto &DefaultCase = [&NewFn, &CI]() -> void {
// Handle generic mangling change, but nothing else
assert(
(CI->getCalledFunction()->getName() != NewFn->getName()) &&
"Unknown function for CallInst upgrade and isn't just a name change");
CI->setCalledFunction(NewFn);
+ };
+ CallInst *NewCall = nullptr;
+ switch (NewFn->getIntrinsicID()) {
+ default: {
+ DefaultCase();
return;
}
@@ -2346,6 +2381,35 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
NewCall = Builder.CreateCall(NewFn, Args);
break;
}
+
+ case Intrinsic::memcpy:
+ case Intrinsic::memmove:
+ case Intrinsic::memset: {
+ // We have to make sure that the call signature is what we're expecting.
+ // We only want to change the old signatures by removing the alignment arg:
+ // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
+ // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
+ // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
+ // -> @llvm.memset...(i8*, i8, i[32|64], i1)
+ // Note: i8*'s in the above can be any pointer type
+ if (CI->getNumArgOperands() != 5) {
+ DefaultCase();
+ return;
+ }
+ // Remove alignment argument (3), and add alignment attributes to the
+ // dest/src pointers.
+ Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
+ CI->getArgOperand(2), CI->getArgOperand(4)};
+ NewCall = Builder.CreateCall(NewFn, Args);
+ auto *MemCI = cast<MemIntrinsic>(NewCall);
+ // All mem intrinsics support dest alignment.
+ const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
+ MemCI->setDestAlignment(Align->getZExtValue());
+ // Memcpy/Memmove also support source alignment.
+ if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
+ MTI->setSourceAlignment(Align->getZExtValue());
+ break;
+ }
}
assert(NewCall && "Should have either set this variable or returned through "
"the default case");
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 027c0255bce..99795f54138 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -15,6 +15,7 @@
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Statepoint.h"
@@ -83,13 +84,16 @@ CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
MDNode *NoAliasTag) {
Ptr = getCastedInt8PtrValue(Ptr);
- Value *Ops[] = { Ptr, Val, Size, getInt32(Align), getInt1(isVolatile) };
+ Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
Type *Tys[] = { Ptr->getType(), Size->getType() };
Module *M = BB->getParent()->getParent();
Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
CallInst *CI = createCallHelper(TheFn, Ops, this);
-
+
+ if (Align > 0)
+ cast<MemSetInst>(CI)->setDestAlignment(Align);
+
// Set the TBAA info if present.
if (TBAATag)
CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
@@ -99,7 +103,7 @@ CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
if (NoAliasTag)
CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
-
+
return CI;
}
@@ -107,16 +111,20 @@ CallInst *IRBuilderBase::
CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align,
bool isVolatile, MDNode *TBAATag, MDNode *TBAAStructTag,
MDNode *ScopeTag, MDNode *NoAliasTag) {
+ assert((Align == 0 || isPowerOf2_32(Align)) && "Must be 0 or a power of 2");
Dst = getCastedInt8PtrValue(Dst);
Src = getCastedInt8PtrValue(Src);
- Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) };
+ Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
Module *M = BB->getParent()->getParent();
Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
CallInst *CI = createCallHelper(TheFn, Ops, this);
-
+
+ if (Align > 0)
+ cast<MemCpyInst>(CI)->setAlignment(Align);
+
// Set the TBAA info if present.
if (TBAATag)
CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
@@ -130,7 +138,7 @@ CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align,
if (NoAliasTag)
CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
-
+
return CI;
}
@@ -154,8 +162,9 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
CallInst *CI = createCallHelper(TheFn, Ops, this);
// Set the alignment of the pointer args.
- CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
- CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
+ auto *AMCI = cast<AtomicMemCpyInst>(CI);
+ AMCI->setDestAlignment(DstAlign);
+ AMCI->setSourceAlignment(SrcAlign);
// Set the TBAA info if present.
if (TBAATag)
@@ -178,16 +187,21 @@ CallInst *IRBuilderBase::
CreateMemMove(Value *Dst, Value *Src, Value *Size, unsigned Align,
bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
MDNode *NoAliasTag) {
+ assert((Align == 0 || isPowerOf2_32(Align)) && "Must be 0 or a power of 2");
Dst = getCastedInt8PtrValue(Dst);
Src = getCastedInt8PtrValue(Src);
-
- Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) };
+
+ Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
Module *M = BB->getParent()->getParent();
Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
CallInst *CI = createCallHelper(TheFn, Ops, this);
-
+
+ auto *MMI = cast<MemMoveInst>(CI);
+ if (Align > 0)
+ MMI->setAlignment(Align);
+
// Set the TBAA info if present.
if (TBAATag)
CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index f7775be6961..b73f9dfe4d6 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -4048,18 +4048,23 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
case Intrinsic::memcpy:
case Intrinsic::memmove:
case Intrinsic::memset: {
- ConstantInt *AlignCI = dyn_cast<ConstantInt>(CS.getArgOperand(3));
- Assert(AlignCI,
- "alignment argument of memory intrinsics must be a constant int",
- CS);
const auto *MI = cast<MemIntrinsic>(CS.getInstruction());
auto IsValidAlignment = [&](unsigned Alignment) -> bool {
return Alignment == 0 || isPowerOf2_32(Alignment);
};
- Assert(IsValidAlignment(MI->getAlignment()),
- "alignment argument of memory intrinsics must be 0 or a power of 2",
+ Assert(IsValidAlignment(MI->getDestAlignment()),
+ "alignment of arg 0 of memory intrinsic must be 0 or a power of 2",
CS);
- Assert(isa<ConstantInt>(CS.getArgOperand(4)),
+ if (const auto *MTI = dyn_cast<MemTransferInst>(MI)) {
+ Assert(IsValidAlignment(MTI->getSourceAlignment()),
+ "alignment of arg 1 of memory intrinsic must be 0 or a power of 2",
+ CS);
+ // TODO: Remove this assert when we enhance IRBuilder API to create
+ // memcpy/memmove with separate source & dest alignments.
+ Assert(MTI->getSourceAlignment() == MTI->getDestAlignment(),
+ "TEMPORARY: source and dest alignments must be the same");
+ }
+ Assert(isa<ConstantInt>(CS.getArgOperand(3)),
"isvolatile argument of memory intrinsics must be a constant int",
CS);
break;
diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
index 022200986d2..59168aea91e 100644
--- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp
@@ -3476,7 +3476,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
return false;
const char *IntrMemName = isa<MemCpyInst>(II) ? "memcpy" : "memmove";
- return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 2);
+ return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 1);
}
case Intrinsic::memset: {
const MemSetInst *MSI = cast<MemSetInst>(II);
@@ -3492,7 +3492,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
// address spaces.
return false;
- return lowerCallTo(II, "memset", II->getNumArgOperands() - 2);
+ return lowerCallTo(II, "memset", II->getNumArgOperands() - 1);
}
case Intrinsic::sin:
case Intrinsic::cos:
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index 60048d4453d..814236ce83c 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -2352,8 +2352,8 @@ bool ARMFastISel::SelectCall(const Instruction *I,
for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
i != e; ++i) {
// If we're lowering a memory intrinsic instead of a regular call, skip the
- // last two arguments, which shouldn't be passed to the underlying function.
- if (IntrMemName && e-i <= 2)
+ // last argument, which shouldn't be passed to the underlying function.
+ if (IntrMemName && e - i <= 1)
break;
ISD::ArgFlagsTy Flags;
diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp
index 8bbac3ed7cf..65dfbdc4564 100644
--- a/llvm/lib/Target/Mips/MipsFastISel.cpp
+++ b/llvm/lib/Target/Mips/MipsFastISel.cpp
@@ -1628,7 +1628,7 @@ bool MipsFastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
if (!MTI->getLength()->getType()->isIntegerTy(32))
return false;
const char *IntrMemName = isa<MemCpyInst>(II) ? "memcpy" : "memmove";
- return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 2);
+ return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 1);
}
case Intrinsic::memset: {
const MemSetInst *MSI = cast<MemSetInst>(II);
@@ -1637,7 +1637,7 @@ bool MipsFastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
return false;
if (!MSI->getLength()->getType()->isIntegerTy(32))
return false;
- return lowerCallTo(II, "memset", II->getNumArgOperands() - 2);
+ return lowerCallTo(II, "memset", II->getNumArgOperands() - 1);
}
}
return false;
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index 5dae485f4c9..cfd51e6e8a2 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -2726,7 +2726,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255)
return false;
- return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 2);
+ return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 1);
}
case Intrinsic::memset: {
const MemSetInst *MSI = cast<MemSetInst>(II);
@@ -2741,7 +2741,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
if (MSI->getDestAddressSpace() > 255)
return false;
- return lowerCallTo(II, "memset", II->getNumArgOperands() - 2);
+ return lowerCallTo(II, "memset", II->getNumArgOperands() - 1);
}
case Intrinsic::stackprotector: {
// Emit code to store the stack guard onto the stack.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 31dea892178..32821e6d9de 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -189,6 +189,7 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
unsigned MinAlign = std::min(DstAlign, SrcAlign);
unsigned CopyAlign = MI->getAlignment();
+ // FIXME: Check & simplify source & dest alignments separately
if (CopyAlign < MinAlign) {
MI->setAlignment(MinAlign);
return MI;
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 5f6fa65abd1..9c90d27d6d5 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1382,20 +1382,19 @@ void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
Value *LenShadow = IRB.CreateMul(
I.getLength(),
ConstantInt::get(I.getLength()->getType(), DFSF.DFS.ShadowWidth / 8));
- Value *AlignShadow;
- if (ClPreserveAlignment) {
- AlignShadow = IRB.CreateMul(I.getAlignmentCst(),
- ConstantInt::get(I.getAlignmentType(),
- DFSF.DFS.ShadowWidth / 8));
- } else {
- AlignShadow = ConstantInt::get(I.getAlignmentType(),
- DFSF.DFS.ShadowWidth / 8);
- }
Type *Int8Ptr = Type::getInt8PtrTy(*DFSF.DFS.Ctx);
DestShadow = IRB.CreateBitCast(DestShadow, Int8Ptr);
SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr);
- IRB.CreateCall(I.getCalledValue(), {DestShadow, SrcShadow, LenShadow,
- AlignShadow, I.getVolatileCst()});
+ auto *MTI = cast<MemTransferInst>(
+ IRB.CreateCall(I.getCalledValue(),
+ {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
+ // FIXME: Set the source & dest alignments of MTI based on the separate
+ // source & dest alignments of I
+ if (ClPreserveAlignment) {
+ MTI->setAlignment(I.getAlignment() * (DFSF.DFS.ShadowWidth / 8));
+ } else {
+ MTI->setAlignment(DFSF.DFS.ShadowWidth / 8);
+ }
}
void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
OpenPOWER on IntegriCloud