diff options
Diffstat (limited to 'llvm/lib')
-rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/IR/AutoUpgrade.cpp | 70 | ||||
-rw-r--r-- | llvm/lib/IR/IRBuilder.cpp | 36 | ||||
-rw-r--r-- | llvm/lib/IR/Verifier.cpp | 19 | ||||
-rw-r--r-- | llvm/lib/Target/AArch64/AArch64FastISel.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/Target/ARM/ARMFastISel.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/Target/Mips/MipsFastISel.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/Target/X86/X86FastISel.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp | 1 | ||||
-rw-r--r-- | llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp | 21 |
10 files changed, 127 insertions, 40 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 27af4296552..1295b83fc6c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5017,6 +5017,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Align = 1; // @llvm.memcpy defines 0 and 1 to both mean no alignment. bool isVol = MCI.isVolatile(); bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget()); + // FIXME: Support passing different dest/src alignments to the memcpy DAG + // node. SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, false, isTC, MachinePointerInfo(I.getArgOperand(0)), @@ -5049,6 +5051,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Align = 1; // @llvm.memmove defines 0 and 1 to both mean no alignment. bool isVol = MMI.isVolatile(); bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget()); + // FIXME: Support passing different dest/src alignments to the memmove DAG + // node. SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, isTC, MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1))); diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index bf96e772394..5d219995aed 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -22,6 +22,7 @@ #include "llvm/IR/Function.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" @@ -522,6 +523,37 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { return true; } } + // Updating the memory intrinsics (memcpy/memmove/memset) that have an + // alignment parameter to embedding the alignment as an attribute of + // the pointer args. + if (Name.startswith("memcpy.") && F->arg_size() == 5) { + rename(F); + // Get the types of dest, src, and len + ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy, + ParamTypes); + return true; + } + if (Name.startswith("memmove.") && F->arg_size() == 5) { + rename(F); + // Get the types of dest, src, and len + ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3); + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove, + ParamTypes); + return true; + } + if (Name.startswith("memset.") && F->arg_size() == 5) { + rename(F); + // Get the types of dest, and len + const auto *FT = F->getFunctionType(); + Type *ParamTypes[2] = { + FT->getParamType(0), // Dest + FT->getParamType(2) // len + }; + NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset, + ParamTypes); + return true; + } break; } case 'n': { @@ -2198,14 +2230,17 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { return; } - CallInst *NewCall = nullptr; - switch (NewFn->getIntrinsicID()) { - default: { + const auto &DefaultCase = [&NewFn, &CI]() -> void { // Handle generic mangling change, but nothing else assert( (CI->getCalledFunction()->getName() != NewFn->getName()) && "Unknown function for CallInst upgrade and isn't just a name change"); CI->setCalledFunction(NewFn); + }; + CallInst *NewCall = nullptr; + switch (NewFn->getIntrinsicID()) { + default: { + DefaultCase(); return; } @@ -2346,6 +2381,35 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { NewCall = Builder.CreateCall(NewFn, Args); break; } + + case Intrinsic::memcpy: + case Intrinsic::memmove: + case Intrinsic::memset: { + // We have to make sure that the call signature is what we're expecting. + // We only want to change the old signatures by removing the alignment arg: + // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1) + // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1) + // @llvm.memset...(i8*, i8, i[32|64], i32, i1) + // -> @llvm.memset...(i8*, i8, i[32|64], i1) + // Note: i8*'s in the above can be any pointer type + if (CI->getNumArgOperands() != 5) { + DefaultCase(); + return; + } + // Remove alignment argument (3), and add alignment attributes to the + // dest/src pointers. + Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2), CI->getArgOperand(4)}; + NewCall = Builder.CreateCall(NewFn, Args); + auto *MemCI = cast<MemIntrinsic>(NewCall); + // All mem intrinsics support dest alignment. + const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); + MemCI->setDestAlignment(Align->getZExtValue()); + // Memcpy/Memmove also support source alignment. + if (auto *MTI = dyn_cast<MemTransferInst>(MemCI)) + MTI->setSourceAlignment(Align->getZExtValue()); + break; + } } assert(NewCall && "Should have either set this variable or returned through " "the default case"); diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp index 027c0255bce..99795f54138 100644 --- a/llvm/lib/IR/IRBuilder.cpp +++ b/llvm/lib/IR/IRBuilder.cpp @@ -15,6 +15,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Statepoint.h" @@ -83,13 +84,16 @@ CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { Ptr = getCastedInt8PtrValue(Ptr); - Value *Ops[] = { Ptr, Val, Size, getInt32(Align), getInt1(isVolatile) }; + Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)}; Type *Tys[] = { Ptr->getType(), Size->getType() }; Module *M = BB->getParent()->getParent(); Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys); CallInst *CI = createCallHelper(TheFn, Ops, this); - + + if (Align > 0) + cast<MemSetInst>(CI)->setDestAlignment(Align); + // Set the TBAA info if present. if (TBAATag) CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); @@ -99,7 +103,7 @@ CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align, if (NoAliasTag) CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); - + return CI; } @@ -107,16 +111,20 @@ CallInst *IRBuilderBase:: CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align, bool isVolatile, MDNode *TBAATag, MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) { + assert((Align == 0 || isPowerOf2_32(Align)) && "Must be 0 or a power of 2"); Dst = getCastedInt8PtrValue(Dst); Src = getCastedInt8PtrValue(Src); - Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) }; + Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; Module *M = BB->getParent()->getParent(); Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys); CallInst *CI = createCallHelper(TheFn, Ops, this); - + + if (Align > 0) + cast<MemCpyInst>(CI)->setAlignment(Align); + // Set the TBAA info if present. if (TBAATag) CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); @@ -130,7 +138,7 @@ CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align, if (NoAliasTag) CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag); - + return CI; } @@ -154,8 +162,9 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy( CallInst *CI = createCallHelper(TheFn, Ops, this); // Set the alignment of the pointer args. - CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign)); - CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign)); + auto *AMCI = cast<AtomicMemCpyInst>(CI); + AMCI->setDestAlignment(DstAlign); + AMCI->setSourceAlignment(SrcAlign); // Set the TBAA info if present. if (TBAATag) @@ -178,16 +187,21 @@ CallInst *IRBuilderBase:: CreateMemMove(Value *Dst, Value *Src, Value *Size, unsigned Align, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) { + assert((Align == 0 || isPowerOf2_32(Align)) && "Must be 0 or a power of 2"); Dst = getCastedInt8PtrValue(Dst); Src = getCastedInt8PtrValue(Src); - - Value *Ops[] = { Dst, Src, Size, getInt32(Align), getInt1(isVolatile) }; + + Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)}; Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() }; Module *M = BB->getParent()->getParent(); Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys); CallInst *CI = createCallHelper(TheFn, Ops, this); - + + auto *MMI = cast<MemMoveInst>(CI); + if (Align > 0) + MMI->setAlignment(Align); + // Set the TBAA info if present. if (TBAATag) CI->setMetadata(LLVMContext::MD_tbaa, TBAATag); diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index f7775be6961..b73f9dfe4d6 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -4048,18 +4048,23 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) { case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: { - ConstantInt *AlignCI = dyn_cast<ConstantInt>(CS.getArgOperand(3)); - Assert(AlignCI, - "alignment argument of memory intrinsics must be a constant int", - CS); const auto *MI = cast<MemIntrinsic>(CS.getInstruction()); auto IsValidAlignment = [&](unsigned Alignment) -> bool { return Alignment == 0 || isPowerOf2_32(Alignment); }; - Assert(IsValidAlignment(MI->getAlignment()), - "alignment argument of memory intrinsics must be 0 or a power of 2", + Assert(IsValidAlignment(MI->getDestAlignment()), + "alignment of arg 0 of memory intrinsic must be 0 or a power of 2", CS); - Assert(isa<ConstantInt>(CS.getArgOperand(4)), + if (const auto *MTI = dyn_cast<MemTransferInst>(MI)) { + Assert(IsValidAlignment(MTI->getSourceAlignment()), + "alignment of arg 1 of memory intrinsic must be 0 or a power of 2", + CS); + // TODO: Remove this assert when we enhance IRBuilder API to create + // memcpy/memmove with separate source & dest alignments. + Assert(MTI->getSourceAlignment() == MTI->getDestAlignment(), + "TEMPORARY: source and dest alignments must be the same"); + } + Assert(isa<ConstantInt>(CS.getArgOperand(3)), "isvolatile argument of memory intrinsics must be a constant int", CS); break; diff --git a/llvm/lib/Target/AArch64/AArch64FastISel.cpp b/llvm/lib/Target/AArch64/AArch64FastISel.cpp index 022200986d2..59168aea91e 100644 --- a/llvm/lib/Target/AArch64/AArch64FastISel.cpp +++ b/llvm/lib/Target/AArch64/AArch64FastISel.cpp @@ -3476,7 +3476,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { return false; const char *IntrMemName = isa<MemCpyInst>(II) ? "memcpy" : "memmove"; - return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 2); + return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 1); } case Intrinsic::memset: { const MemSetInst *MSI = cast<MemSetInst>(II); @@ -3492,7 +3492,7 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { // address spaces. return false; - return lowerCallTo(II, "memset", II->getNumArgOperands() - 2); + return lowerCallTo(II, "memset", II->getNumArgOperands() - 1); } case Intrinsic::sin: case Intrinsic::cos: diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp index 60048d4453d..814236ce83c 100644 --- a/llvm/lib/Target/ARM/ARMFastISel.cpp +++ b/llvm/lib/Target/ARM/ARMFastISel.cpp @@ -2352,8 +2352,8 @@ bool ARMFastISel::SelectCall(const Instruction *I, for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end(); i != e; ++i) { // If we're lowering a memory intrinsic instead of a regular call, skip the - // last two arguments, which shouldn't be passed to the underlying function. - if (IntrMemName && e-i <= 2) + // last argument, which shouldn't be passed to the underlying function. + if (IntrMemName && e - i <= 1) break; ISD::ArgFlagsTy Flags; diff --git a/llvm/lib/Target/Mips/MipsFastISel.cpp b/llvm/lib/Target/Mips/MipsFastISel.cpp index 8bbac3ed7cf..65dfbdc4564 100644 --- a/llvm/lib/Target/Mips/MipsFastISel.cpp +++ b/llvm/lib/Target/Mips/MipsFastISel.cpp @@ -1628,7 +1628,7 @@ bool MipsFastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { if (!MTI->getLength()->getType()->isIntegerTy(32)) return false; const char *IntrMemName = isa<MemCpyInst>(II) ? "memcpy" : "memmove"; - return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 2); + return lowerCallTo(II, IntrMemName, II->getNumArgOperands() - 1); } case Intrinsic::memset: { const MemSetInst *MSI = cast<MemSetInst>(II); @@ -1637,7 +1637,7 @@ bool MipsFastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { return false; if (!MSI->getLength()->getType()->isIntegerTy(32)) return false; - return lowerCallTo(II, "memset", II->getNumArgOperands() - 2); + return lowerCallTo(II, "memset", II->getNumArgOperands() - 1); } } return false; diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp index 5dae485f4c9..cfd51e6e8a2 100644 --- a/llvm/lib/Target/X86/X86FastISel.cpp +++ b/llvm/lib/Target/X86/X86FastISel.cpp @@ -2726,7 +2726,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { if (MCI->getSourceAddressSpace() > 255 || MCI->getDestAddressSpace() > 255) return false; - return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 2); + return lowerCallTo(II, "memcpy", II->getNumArgOperands() - 1); } case Intrinsic::memset: { const MemSetInst *MSI = cast<MemSetInst>(II); @@ -2741,7 +2741,7 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { if (MSI->getDestAddressSpace() > 255) return false; - return lowerCallTo(II, "memset", II->getNumArgOperands() - 2); + return lowerCallTo(II, "memset", II->getNumArgOperands() - 1); } case Intrinsic::stackprotector: { // Emit code to store the stack guard onto the stack. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 31dea892178..32821e6d9de 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -189,6 +189,7 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) { unsigned MinAlign = std::min(DstAlign, SrcAlign); unsigned CopyAlign = MI->getAlignment(); + // FIXME: Check & simplify source & dest alignments separately if (CopyAlign < MinAlign) { MI->setAlignment(MinAlign); return MI; diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp index 5f6fa65abd1..9c90d27d6d5 100644 --- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp @@ -1382,20 +1382,19 @@ void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) { Value *LenShadow = IRB.CreateMul( I.getLength(), ConstantInt::get(I.getLength()->getType(), DFSF.DFS.ShadowWidth / 8)); - Value *AlignShadow; - if (ClPreserveAlignment) { - AlignShadow = IRB.CreateMul(I.getAlignmentCst(), - ConstantInt::get(I.getAlignmentType(), - DFSF.DFS.ShadowWidth / 8)); - } else { - AlignShadow = ConstantInt::get(I.getAlignmentType(), - DFSF.DFS.ShadowWidth / 8); - } Type *Int8Ptr = Type::getInt8PtrTy(*DFSF.DFS.Ctx); DestShadow = IRB.CreateBitCast(DestShadow, Int8Ptr); SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr); - IRB.CreateCall(I.getCalledValue(), {DestShadow, SrcShadow, LenShadow, - AlignShadow, I.getVolatileCst()}); + auto *MTI = cast<MemTransferInst>( + IRB.CreateCall(I.getCalledValue(), + {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()})); + // FIXME: Set the source & dest alignments of MTI based on the separate + // source & dest alignments of I + if (ClPreserveAlignment) { + MTI->setAlignment(I.getAlignment() * (DFSF.DFS.ShadowWidth / 8)); + } else { + MTI->setAlignment(DFSF.DFS.ShadowWidth / 8); + } } void DFSanVisitor::visitReturnInst(ReturnInst &RI) { |