From 53270d0fa61eb95a3f948dbcf5d65373abc86648 Mon Sep 17 00:00:00 2001 From: "Ivan A. Kosarev" Date: Fri, 16 Feb 2018 10:10:29 +0000 Subject: [Transforms] Propagate TBAA info in SROA Now that we have the new TBAA metadata format that is capable of representing accesses to aggregates, we can propagate TBAA access tags from memory setting and transferring intrinsics to load and store instructions and vice versa. Since SROA produces lots of new loads and stores on optimized builds, this change significantly decreases the share of undecorated memory accesses on such builds. Differential Revision: https://reviews.llvm.org/D41563 llvm-svn: 325329 --- llvm/lib/Transforms/Scalar/SROA.cpp | 83 +++++++++++++++++++++++++++---------- 1 file changed, 61 insertions(+), 22 deletions(-) (limited to 'llvm/lib/Transforms/Scalar/SROA.cpp') diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 6089fc81f87..c01f3321fc2 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -2430,6 +2430,9 @@ private: Value *OldOp = LI.getOperand(0); assert(OldOp == OldPtr); + AAMDNodes AATags; + LI.getAAMetadata(AATags); + unsigned AS = LI.getPointerAddressSpace(); Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8) @@ -2448,6 +2451,8 @@ private: TargetTy->isIntegerTy()))) { LoadInst *NewLI = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), LI.isVolatile(), LI.getName()); + if (AATags) + NewLI->setAAMetadata(AATags); if (LI.isVolatile()) NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); @@ -2483,6 +2488,8 @@ private: LoadInst *NewLI = IRB.CreateAlignedLoad(getNewAllocaSlicePtr(IRB, LTy), getSliceAlign(TargetTy), LI.isVolatile(), LI.getName()); + if (AATags) + NewLI->setAAMetadata(AATags); if (LI.isVolatile()) NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); @@ -2523,7 +2530,8 @@ private: return !LI.isVolatile() && !IsPtrAdjusted; } - bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp) { + bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp, + AAMDNodes AATags) { if (V->getType() != VecTy) { unsigned BeginIndex = getIndex(NewBeginOffset); unsigned EndIndex = getIndex(NewEndOffset); @@ -2541,14 +2549,15 @@ private: V = insertVector(IRB, Old, V, BeginIndex, "vec"); } StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); + if (AATags) + Store->setAAMetadata(AATags); Pass.DeadInsts.insert(&SI); - (void)Store; DEBUG(dbgs() << " to: " << *Store << "\n"); return true; } - bool rewriteIntegerStore(Value *V, StoreInst &SI) { + bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) { assert(IntTy && "We cannot extract an integer from the alloca"); assert(!SI.isVolatile()); if (DL.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) { @@ -2562,6 +2571,8 @@ private: V = convertValue(DL, IRB, V, NewAllocaTy); StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); Store->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access); + if (AATags) + Store->setAAMetadata(AATags); Pass.DeadInsts.insert(&SI); DEBUG(dbgs() << " to: " << *Store << "\n"); return true; @@ -2572,6 +2583,9 @@ private: Value *OldOp = SI.getOperand(1); assert(OldOp == OldPtr); + AAMDNodes AATags; + SI.getAAMetadata(AATags); + Value *V = SI.getValueOperand(); // Strip all inbounds GEPs and pointer casts to try to dig out any root @@ -2593,9 +2607,9 @@ private: } if (VecTy) - return rewriteVectorizedStoreInst(V, SI, OldOp); + return rewriteVectorizedStoreInst(V, SI, OldOp, AATags); if (IntTy && V->getType()->isIntegerTy()) - return rewriteIntegerStore(V, SI); + return rewriteIntegerStore(V, SI, AATags); const bool IsStorePastEnd = DL.getTypeStoreSize(V->getType()) > SliceSize; StoreInst *NewSI; @@ -2626,6 +2640,8 @@ private: SI.isVolatile()); } NewSI->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access); + if (AATags) + NewSI->setAAMetadata(AATags); if (SI.isVolatile()) NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); Pass.DeadInsts.insert(&SI); @@ -2673,6 +2689,9 @@ private: DEBUG(dbgs() << " original: " << II << "\n"); assert(II.getRawDest() == OldPtr); + AAMDNodes AATags; + II.getAAMetadata(AATags); + // If the memset has a variable size, it cannot be split, just adjust the // pointer to the new alloca. if (!isa(II.getLength())) { @@ -2704,7 +2723,8 @@ private: CallInst *New = IRB.CreateMemSet( getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size, getSliceAlign(), II.isVolatile()); - (void)New; + if (AATags) + New->setAAMetadata(AATags); DEBUG(dbgs() << " to: " << *New << "\n"); return false; } @@ -2767,9 +2787,10 @@ private: V = convertValue(DL, IRB, V, AllocaTy); } - Value *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), - II.isVolatile()); - (void)New; + StoreInst *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), + II.isVolatile()); + if (AATags) + New->setAAMetadata(AATags); DEBUG(dbgs() << " to: " << *New << "\n"); return !II.isVolatile(); } @@ -2780,6 +2801,9 @@ private: DEBUG(dbgs() << " original: " << II << "\n"); + AAMDNodes AATags; + II.getAAMetadata(AATags); + bool IsDest = &II.getRawDestUse() == OldUse; assert((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)); @@ -2870,7 +2894,8 @@ private: CallInst *New = IRB.CreateMemCpy( IsDest ? OurPtr : OtherPtr, IsDest ? OtherPtr : OurPtr, Size, MinAlign(SliceAlign, OtherAlign), II.isVolatile()); - (void)New; + if (AATags) + New->setAAMetadata(AATags); DEBUG(dbgs() << " to: " << *New << "\n"); return false; } @@ -2919,8 +2944,11 @@ private: uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract"); } else { - Src = - IRB.CreateAlignedLoad(SrcPtr, SrcAlign, II.isVolatile(), "copyload"); + LoadInst *Load = IRB.CreateAlignedLoad(SrcPtr, SrcAlign, II.isVolatile(), + "copyload"); + if (AATags) + Load->setAAMetadata(AATags); + Src = Load; } if (VecTy && !IsWholeAlloca && IsDest) { @@ -2938,7 +2966,8 @@ private: StoreInst *Store = cast( IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile())); - (void)Store; + if (AATags) + Store->setAAMetadata(AATags); DEBUG(dbgs() << " to: " << *Store << "\n"); return !II.isVolatile(); } @@ -3157,8 +3186,10 @@ private: }; struct LoadOpSplitter : public OpSplitter { - LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr) - : OpSplitter(InsertionPoint, Ptr) {} + AAMDNodes AATags; + + LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, AAMDNodes AATags) + : OpSplitter(InsertionPoint, Ptr), AATags(AATags) {} /// Emit a leaf load of a single value. This is called at the leaves of the /// recursive emission to actually load values. @@ -3167,7 +3198,9 @@ private: // Load the single value and insert it using the indices. Value *GEP = IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep"); - Value *Load = IRB.CreateLoad(GEP, Name + ".load"); + LoadInst *Load = IRB.CreateLoad(GEP, Name + ".load"); + if (AATags) + Load->setAAMetadata(AATags); Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert"); DEBUG(dbgs() << " to: " << *Load << "\n"); } @@ -3180,7 +3213,9 @@ private: // We have an aggregate being loaded, split it apart. DEBUG(dbgs() << " original: " << LI << "\n"); - LoadOpSplitter Splitter(&LI, *U); + AAMDNodes AATags; + LI.getAAMetadata(AATags); + LoadOpSplitter Splitter(&LI, *U, AATags); Value *V = UndefValue::get(LI.getType()); Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca"); LI.replaceAllUsesWith(V); @@ -3189,8 +3224,9 @@ private: } struct StoreOpSplitter : public OpSplitter { - StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr) - : OpSplitter(InsertionPoint, Ptr) {} + StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, AAMDNodes AATags) + : OpSplitter(InsertionPoint, Ptr), AATags(AATags) {} + AAMDNodes AATags; /// Emit a leaf store of a single value. This is called at the leaves of the /// recursive emission to actually produce stores. @@ -3204,8 +3240,9 @@ private: IRB.CreateExtractValue(Agg, Indices, Name + ".extract"); Value *InBoundsGEP = IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep"); - Value *Store = IRB.CreateStore(ExtractValue, InBoundsGEP); - (void)Store; + StoreInst *Store = IRB.CreateStore(ExtractValue, InBoundsGEP); + if (AATags) + Store->setAAMetadata(AATags); DEBUG(dbgs() << " to: " << *Store << "\n"); } }; @@ -3219,7 +3256,9 @@ private: // We have an aggregate being stored, split it apart. DEBUG(dbgs() << " original: " << SI << "\n"); - StoreOpSplitter Splitter(&SI, *U); + AAMDNodes AATags; + SI.getAAMetadata(AATags); + StoreOpSplitter Splitter(&SI, *U, AATags); Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca"); SI.eraseFromParent(); return true; -- cgit v1.2.3