diff options
| -rw-r--r-- | polly/include/polly/CodeGen/BlockGenerators.h | 182 | ||||
| -rw-r--r-- | polly/include/polly/CodeGen/IslNodeBuilder.h | 17 | ||||
| -rw-r--r-- | polly/lib/CodeGen/BlockGenerators.cpp | 168 | ||||
| -rw-r--r-- | polly/lib/CodeGen/IslNodeBuilder.cpp | 35 |
4 files changed, 251 insertions, 151 deletions
diff --git a/polly/include/polly/CodeGen/BlockGenerators.h b/polly/include/polly/CodeGen/BlockGenerators.h index b78723461cb..aac47fcb8c3 100644 --- a/polly/include/polly/CodeGen/BlockGenerators.h +++ b/polly/include/polly/CodeGen/BlockGenerators.h @@ -23,6 +23,7 @@ #include <vector> struct isl_ast_build; +struct isl_id_to_ast_expr; namespace llvm { class Pass; @@ -102,12 +103,17 @@ public: /// This copies the entire basic block and updates references to old values /// with references to new values, as defined by GlobalMap. /// - /// @param Stmt The block statement to code generate. - /// @param GlobalMap A mapping from old values to their new values - /// (for values recalculated in the new ScoP, but not - /// within this basic block). - /// @param LTS A map from old loops to new induction variables as SCEVs. - void copyStmt(ScopStmt &Stmt, ValueMapT &GlobalMap, LoopToScevMapT <S); + /// @param Stmt The block statement to code generate. + /// @param GlobalMap A mapping from old values to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param LTS A map from old loops to new induction variables as + /// SCEVs. + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. + void copyStmt(ScopStmt &Stmt, ValueMapT &GlobalMap, LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses); /// @brief Finalize the code generation for the SCoP @p S. /// @@ -274,14 +280,19 @@ protected: /// @param BB The basic block to code generate. /// @param BBMap A mapping from old values to their new values in this /// block. - /// @param GlobalMap A mapping from old values to their new values - /// (for values recalculated in the new ScoP, but not - /// within this basic block). - /// @param LTS A map from old loops to new induction variables as SCEVs. + /// @param GlobalMap A mapping from old values to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param LTS A map from old loops to new induction variables as + /// SCEVs. + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. /// /// @returns The copy of the basic block. BasicBlock *copyBB(ScopStmt &Stmt, BasicBlock *BB, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + ValueMapT &GlobalMap, LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses); /// @brief Copy the given basic block. /// @@ -290,12 +301,17 @@ protected: /// @param BBCopy The new basic block to generate code in. /// @param BBMap A mapping from old values to their new values in this /// block. - /// @param GlobalMap A mapping from old values to their new values - /// (for values recalculated in the new ScoP, but not - /// within this basic block). - /// @param LTS A map from old loops to new induction variables as SCEVs. + /// @param GlobalMap A mapping from old values to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param LTS A map from old loops to new induction variables as + /// SCEVs. + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. void copyBB(ScopStmt &Stmt, BasicBlock *BB, BasicBlock *BBCopy, - ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S); + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses); /// @brief Return the alloca for @p ScalarBase in @p Map. /// @@ -394,21 +410,30 @@ protected: /// @return The innermost loop that surrounds the instruction. Loop *getLoopForInst(const Instruction *Inst); - /// @brief Get the new operand address according to access relation of @p MA. - Value *getNewAccessOperand(ScopStmt &Stmt, const MemoryAccess &MA); - /// @brief Generate the operand address + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. Value *generateLocationAccessed(ScopStmt &Stmt, const Instruction *Inst, const Value *Pointer, ValueMapT &BBMap, - ValueMapT &GlobalMap, LoopToScevMapT <S); + ValueMapT &GlobalMap, LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses); + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. Value *generateScalarLoad(ScopStmt &Stmt, const LoadInst *load, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S); + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses); + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. void generateScalarStore(ScopStmt &Stmt, const StoreInst *store, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S); + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses); /// @brief Copy a single PHI instruction. /// @@ -426,20 +451,23 @@ protected: /// This copies a single Instruction and updates references to old values /// with references to new values, as defined by GlobalMap and BBMap. /// - /// @param Stmt The statement to code generate. - /// @param Inst The instruction to copy. - /// @param BBMap A mapping from old values to their new values - /// (for values recalculated within this basic block). - /// @param GlobalMap A mapping from old values to their new values - /// (for values recalculated in the new ScoP, but not - /// within this basic block). - /// @param LTS A mapping from loops virtual canonical induction - /// variable to their new values - /// (for values recalculated in the new ScoP, but not - /// within this basic block). + /// @param Stmt The statement to code generate. + /// @param Inst The instruction to copy. + /// @param BBMap A mapping from old values to their new values + /// (for values recalculated within this basic block). + /// @param GlobalMap A mapping from old values to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param LTS A mapping from loops virtual canonical induction + /// variable to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block). + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S); + LoopToScevMapT <S, isl_id_to_ast_expr *NewAccesses); /// @brief Helper to get the newest version of @p ScalarValue. /// @@ -471,27 +499,31 @@ public: /// instructions, but e.g. for address calculation instructions we currently /// generate scalar instructions for each vector lane. /// - /// @param BlockGen A block generator object used as parent. - /// @param Stmt The statement to code generate. - /// @param GlobalMaps A vector of maps that define for certain Values - /// referenced from the original code new Values they should - /// be replaced with. Each map in the vector of maps is - /// used for one vector lane. The number of elements in the - /// vector defines the width of the generated vector - /// instructions. - /// @param VLTS A mapping from loops virtual canonical induction - /// variable to their new values - /// (for values recalculated in the new ScoP, but not - /// within this basic block), one for each lane. - /// @param Schedule A map from the statement to a schedule where the - /// innermost dimension is the dimension of the innermost - /// loop containing the statemenet. + /// @param BlockGen A block generator object used as parent. + /// @param Stmt The statement to code generate. + /// @param GlobalMaps A vector of maps that define for certain Values + /// referenced from the original code new Values they + /// should be replaced with. Each map in the vector of maps + /// is used for one vector lane. The number of elements in + /// the vector defines the width of the generated vector + /// instructions. + /// @param VLTS A mapping from loops virtual canonical induction + /// variable to their new values + /// (for values recalculated in the new ScoP, but not + /// within this basic block), one for each lane. + /// @param Schedule A map from the statement to a schedule where the + /// innermost dimension is the dimension of the innermost + /// loop containing the statemenet. + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. static void generate(BlockGenerator &BlockGen, ScopStmt &Stmt, VectorValueMapT &GlobalMaps, std::vector<LoopToScevMapT> &VLTS, - __isl_keep isl_map *Schedule) { + __isl_keep isl_map *Schedule, + __isl_keep isl_id_to_ast_expr *NewAccesses) { VectorBlockGenerator Generator(BlockGen, GlobalMaps, VLTS, Schedule); - Generator.copyStmt(Stmt); + Generator.copyStmt(Stmt, NewAccesses); } private: @@ -550,8 +582,12 @@ private: /// vector. By default we would do only positive /// strides. /// + /// @param NewAccesses A map from memory access ids to new ast + /// expressions, which may contain new access + /// expressions for certain memory accesses. Value *generateStrideOneLoad(ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, + __isl_keep isl_id_to_ast_expr *NewAccesses, bool NegativeStride); /// @brief Load a vector initialized from a single scalar in memory @@ -564,8 +600,12 @@ private: /// %splat = shufflevector <1 x double> %splat_one, <1 x /// double> %splat_one, <4 x i32> zeroinitializer /// + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. Value *generateStrideZeroLoad(ScopStmt &Stmt, const LoadInst *Load, - ValueMapT &BBMap); + ValueMapT &BBMap, + __isl_keep isl_id_to_ast_expr *NewAccesses); /// @brief Load a vector from scalars distributed in memory /// @@ -578,11 +618,19 @@ private: /// %scalar 2 = load double* %p_2 /// %vec_2 = insertelement <2 x double> %vec_1, double %scalar_1, i32 1 /// + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. Value *generateUnknownStrideLoad(ScopStmt &Stmt, const LoadInst *Load, - VectorValueMapT &ScalarMaps); + VectorValueMapT &ScalarMaps, + __isl_keep isl_id_to_ast_expr *NewAccesses); + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. void generateLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + VectorValueMapT &ScalarMaps, + __isl_keep isl_id_to_ast_expr *NewAccesses); void copyUnaryInst(ScopStmt &Stmt, const UnaryInstruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); @@ -590,21 +638,36 @@ private: void copyBinaryInst(ScopStmt &Stmt, const BinaryOperator *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. void copyStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps); + VectorValueMapT &ScalarMaps, + __isl_keep isl_id_to_ast_expr *NewAccesses); + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. void copyInstScalarized(ScopStmt &Stmt, const Instruction *Inst, - ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, + __isl_keep isl_id_to_ast_expr *NewAccesses); bool extractScalarValues(const Instruction *Inst, ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); bool hasVectorOperands(const Instruction *Inst, ValueMapT &VectorMap); + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. void copyInstruction(ScopStmt &Stmt, const Instruction *Inst, - ValueMapT &VectorMap, VectorValueMapT &ScalarMaps); + ValueMapT &VectorMap, VectorValueMapT &ScalarMaps, + __isl_keep isl_id_to_ast_expr *NewAccesses); - void copyStmt(ScopStmt &Stmt); + /// @param NewAccesses A map from memory access ids to new ast expressions, + /// which may contain new access expressions for certain + /// memory accesses. + void copyStmt(ScopStmt &Stmt, __isl_keep isl_id_to_ast_expr *NewAccesses); }; /// @brief Generator for new versions of polyhedral region statements. @@ -626,7 +689,8 @@ public: /// (for values recalculated in the new ScoP, but not /// within this basic block). /// @param LTS A map from old loops to new induction variables as SCEVs. - void copyStmt(ScopStmt &Stmt, ValueMapT &GlobalMap, LoopToScevMapT <S); + void copyStmt(ScopStmt &Stmt, ValueMapT &GlobalMap, LoopToScevMapT <S, + __isl_keep isl_id_to_ast_expr *IdToAstExp); /// @brief An empty destructor virtual ~RegionGenerator(){}; diff --git a/polly/include/polly/CodeGen/IslNodeBuilder.h b/polly/include/polly/CodeGen/IslNodeBuilder.h index 1adb86e7ef2..c38d64b6d39 100644 --- a/polly/include/polly/CodeGen/IslNodeBuilder.h +++ b/polly/include/polly/CodeGen/IslNodeBuilder.h @@ -23,6 +23,7 @@ using namespace polly; using namespace llvm; struct isl_ast_node; +struct isl_ast_build; class IslNodeBuilder { public: @@ -191,6 +192,22 @@ private: /// @param For The FOR isl_ast_node for which code is generated. void createForParallel(__isl_take isl_ast_node *For); + /// @brief Create new access functions for modified memory accesses. + /// + /// In case the access function of one of the memory references in the Stmt + /// has been modified, we generate a new isl_ast_expr that reflects the + /// newly modified access function and return a map that maps from the + /// individual memory references in the statement (identified by their id) + /// to these newly generated ast expressions. + /// + /// @param Build The build to use to generate these expressions. + /// @param Stmt The statement for which to (possibly) generate new access + /// functions. + /// @return A new hash table that contains remappings from memory ids to new + /// access expressions. + __isl_give isl_id_to_ast_expr * + createNewAccesses(ScopStmt *Stmt, __isl_keep isl_ast_build *Build); + /// Generate LLVM-IR that computes the values of the original induction /// variables in function of the newly generated loop induction variables. /// diff --git a/polly/lib/CodeGen/BlockGenerators.cpp b/polly/lib/CodeGen/BlockGenerators.cpp index ca9fcd88507..1b5e7ccd63d 100644 --- a/polly/lib/CodeGen/BlockGenerators.cpp +++ b/polly/lib/CodeGen/BlockGenerators.cpp @@ -178,38 +178,21 @@ void BlockGenerator::copyInstScalar(ScopStmt &Stmt, const Instruction *Inst, NewInst->setName("p_" + Inst->getName()); } -Value *BlockGenerator::getNewAccessOperand(ScopStmt &Stmt, - const MemoryAccess &MA) { - isl_pw_multi_aff *PWAccRel; - isl_union_map *Schedule; - isl_ast_expr *Expr; - isl_ast_build *Build = Stmt.getAstBuild(); - - assert(ExprBuilder && Build && - "Cannot generate new value without IslExprBuilder!"); - - Schedule = isl_ast_build_get_schedule(Build); - PWAccRel = MA.applyScheduleToAccessRelation(Schedule); - - Expr = isl_ast_build_access_from_pw_multi_aff(Build, PWAccRel); - Expr = isl_ast_expr_address_of(Expr); - - return ExprBuilder->create(Expr); -} - Value *BlockGenerator::generateLocationAccessed( ScopStmt &Stmt, const Instruction *Inst, const Value *Pointer, - ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S) { + ValueMapT &BBMap, ValueMapT &GlobalMap, LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { const MemoryAccess &MA = Stmt.getAccessFor(Inst); - Value *NewPointer; - if (MA.hasNewAccessRelation()) - NewPointer = getNewAccessOperand(Stmt, MA); - else - NewPointer = - getNewValue(Stmt, Pointer, BBMap, GlobalMap, LTS, getLoopForInst(Inst)); + isl_ast_expr *AccessExpr = isl_id_to_ast_expr_get(NewAccesses, MA.getId()); - return NewPointer; + if (AccessExpr) { + AccessExpr = isl_ast_expr_address_of(AccessExpr); + return ExprBuilder->create(AccessExpr); + } + + return getNewValue(Stmt, Pointer, BBMap, GlobalMap, LTS, + getLoopForInst(Inst)); } Loop *BlockGenerator::getLoopForInst(const llvm::Instruction *Inst) { @@ -219,10 +202,11 @@ Loop *BlockGenerator::getLoopForInst(const llvm::Instruction *Inst) { Value *BlockGenerator::generateScalarLoad(ScopStmt &Stmt, const LoadInst *Load, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S) { + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { const Value *Pointer = Load->getPointerOperand(); - Value *NewPointer = - generateLocationAccessed(Stmt, Load, Pointer, BBMap, GlobalMap, LTS); + Value *NewPointer = generateLocationAccessed(Stmt, Load, Pointer, BBMap, + GlobalMap, LTS, NewAccesses); Value *ScalarLoad = Builder.CreateAlignedLoad( NewPointer, Load->getAlignment(), Load->getName() + "_p_scalar_"); return ScalarLoad; @@ -230,10 +214,11 @@ Value *BlockGenerator::generateScalarLoad(ScopStmt &Stmt, const LoadInst *Load, void BlockGenerator::generateScalarStore(ScopStmt &Stmt, const StoreInst *Store, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S) { + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { const Value *Pointer = Store->getPointerOperand(); - Value *NewPointer = - generateLocationAccessed(Stmt, Store, Pointer, BBMap, GlobalMap, LTS); + Value *NewPointer = generateLocationAccessed(Stmt, Store, Pointer, BBMap, + GlobalMap, LTS, NewAccesses); Value *ValueOperand = getNewValue(Stmt, Store->getValueOperand(), BBMap, GlobalMap, LTS, getLoopForInst(Store)); @@ -242,7 +227,8 @@ void BlockGenerator::generateScalarStore(ScopStmt &Stmt, const StoreInst *Store, void BlockGenerator::copyInstruction(ScopStmt &Stmt, const Instruction *Inst, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S) { + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { // First check for possible scalar dependences for this instruction. generateScalarLoads(Stmt, Inst, BBMap); @@ -261,7 +247,8 @@ void BlockGenerator::copyInstruction(ScopStmt &Stmt, const Instruction *Inst, } if (const LoadInst *Load = dyn_cast<LoadInst>(Inst)) { - Value *NewLoad = generateScalarLoad(Stmt, Load, BBMap, GlobalMap, LTS); + Value *NewLoad = + generateScalarLoad(Stmt, Load, BBMap, GlobalMap, LTS, NewAccesses); // Compute NewLoad before its insertion in BBMap to make the insertion // deterministic. BBMap[Load] = NewLoad; @@ -269,7 +256,7 @@ void BlockGenerator::copyInstruction(ScopStmt &Stmt, const Instruction *Inst, } if (const StoreInst *Store = dyn_cast<StoreInst>(Inst)) { - generateScalarStore(Stmt, Store, BBMap, GlobalMap, LTS); + generateScalarStore(Stmt, Store, BBMap, GlobalMap, LTS, NewAccesses); return; } @@ -306,14 +293,15 @@ void BlockGenerator::copyInstruction(ScopStmt &Stmt, const Instruction *Inst, } void BlockGenerator::copyStmt(ScopStmt &Stmt, ValueMapT &GlobalMap, - LoopToScevMapT <S) { + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { assert(Stmt.isBlockStmt() && "Only block statements can be copied by the block generator"); ValueMapT BBMap; BasicBlock *BB = Stmt.getBasicBlock(); - copyBB(Stmt, BB, BBMap, GlobalMap, LTS); + copyBB(Stmt, BB, BBMap, GlobalMap, LTS, NewAccesses); } BasicBlock *BlockGenerator::splitBB(BasicBlock *BB) { @@ -325,20 +313,22 @@ BasicBlock *BlockGenerator::splitBB(BasicBlock *BB) { BasicBlock *BlockGenerator::copyBB(ScopStmt &Stmt, BasicBlock *BB, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S) { + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { BasicBlock *CopyBB = splitBB(BB); - copyBB(Stmt, BB, CopyBB, BBMap, GlobalMap, LTS); + copyBB(Stmt, BB, CopyBB, BBMap, GlobalMap, LTS, NewAccesses); return CopyBB; } void BlockGenerator::copyBB(ScopStmt &Stmt, BasicBlock *BB, BasicBlock *CopyBB, ValueMapT &BBMap, ValueMapT &GlobalMap, - LoopToScevMapT <S) { + LoopToScevMapT <S, + isl_id_to_ast_expr *NewAccesses) { Builder.SetInsertPoint(CopyBB->begin()); EntryBB = &CopyBB->getParent()->getEntryBlock(); for (Instruction &Inst : *BB) - copyInstruction(Stmt, &Inst, BBMap, GlobalMap, LTS); + copyInstruction(Stmt, &Inst, BBMap, GlobalMap, LTS, NewAccesses); // After a basic block was copied store all scalars that escape this block // in their alloca. First the scalars that have dependences inside the SCoP, @@ -634,15 +624,16 @@ Type *VectorBlockGenerator::getVectorPtrTy(const Value *Val, int Width) { Value *VectorBlockGenerator::generateStrideOneLoad( ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, - bool NegativeStride = false) { + __isl_keep isl_id_to_ast_expr *NewAccesses, bool NegativeStride = false) { unsigned VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, VectorWidth); unsigned Offset = NegativeStride ? VectorWidth - 1 : 0; Value *NewPointer = nullptr; - NewPointer = generateLocationAccessed(Stmt, Load, Pointer, ScalarMaps[Offset], - GlobalMaps[Offset], VLTS[Offset]); + NewPointer = + generateLocationAccessed(Stmt, Load, Pointer, ScalarMaps[Offset], + GlobalMaps[Offset], VLTS[Offset], NewAccesses); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); LoadInst *VecLoad = @@ -663,13 +654,13 @@ Value *VectorBlockGenerator::generateStrideOneLoad( return VecLoad; } -Value *VectorBlockGenerator::generateStrideZeroLoad(ScopStmt &Stmt, - const LoadInst *Load, - ValueMapT &BBMap) { +Value *VectorBlockGenerator::generateStrideZeroLoad( + ScopStmt &Stmt, const LoadInst *Load, ValueMapT &BBMap, + __isl_keep isl_id_to_ast_expr *NewAccesses) { const Value *Pointer = Load->getPointerOperand(); Type *VectorPtrType = getVectorPtrTy(Pointer, 1); - Value *NewPointer = generateLocationAccessed(Stmt, Load, Pointer, BBMap, - GlobalMaps[0], VLTS[0]); + Value *NewPointer = generateLocationAccessed( + Stmt, Load, Pointer, BBMap, GlobalMaps[0], VLTS[0], NewAccesses); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, Load->getName() + "_p_vec_p"); LoadInst *ScalarLoad = @@ -687,7 +678,10 @@ Value *VectorBlockGenerator::generateStrideZeroLoad(ScopStmt &Stmt, } Value *VectorBlockGenerator::generateUnknownStrideLoad( - ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps) { + ScopStmt &Stmt, const LoadInst *Load, VectorValueMapT &ScalarMaps, + __isl_keep isl_id_to_ast_expr *NewAccesses + + ) { int VectorWidth = getVectorWidth(); const Value *Pointer = Load->getPointerOperand(); VectorType *VectorType = VectorType::get( @@ -696,8 +690,9 @@ Value *VectorBlockGenerator::generateUnknownStrideLoad( Value *Vector = UndefValue::get(VectorType); for (int i = 0; i < VectorWidth; i++) { - Value *NewPointer = generateLocationAccessed( - Stmt, Load, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + Value *NewPointer = + generateLocationAccessed(Stmt, Load, Pointer, ScalarMaps[i], + GlobalMaps[i], VLTS[i], NewAccesses); Value *ScalarLoad = Builder.CreateLoad(NewPointer, Load->getName() + "_p_scalar_"); Vector = Builder.CreateInsertElement( @@ -707,13 +702,13 @@ Value *VectorBlockGenerator::generateUnknownStrideLoad( return Vector; } -void VectorBlockGenerator::generateLoad(ScopStmt &Stmt, const LoadInst *Load, - ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps) { +void VectorBlockGenerator::generateLoad( + ScopStmt &Stmt, const LoadInst *Load, ValueMapT &VectorMap, + VectorValueMapT &ScalarMaps, __isl_keep isl_id_to_ast_expr *NewAccesses) { if (!VectorType::isValidElementType(Load->getType())) { for (int i = 0; i < getVectorWidth(); i++) - ScalarMaps[i][Load] = - generateScalarLoad(Stmt, Load, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + ScalarMaps[i][Load] = generateScalarLoad( + Stmt, Load, ScalarMaps[i], GlobalMaps[i], VLTS[i], NewAccesses); return; } @@ -725,13 +720,13 @@ void VectorBlockGenerator::generateLoad(ScopStmt &Stmt, const LoadInst *Load, Value *NewLoad; if (Access.isStrideZero(isl_map_copy(Schedule))) - NewLoad = generateStrideZeroLoad(Stmt, Load, ScalarMaps[0]); + NewLoad = generateStrideZeroLoad(Stmt, Load, ScalarMaps[0], NewAccesses); else if (Access.isStrideOne(isl_map_copy(Schedule))) - NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, NewAccesses); else if (Access.isStrideX(isl_map_copy(Schedule), -1)) - NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, true); + NewLoad = generateStrideOneLoad(Stmt, Load, ScalarMaps, NewAccesses, true); else - NewLoad = generateUnknownStrideLoad(Stmt, Load, ScalarMaps); + NewLoad = generateUnknownStrideLoad(Stmt, Load, ScalarMaps, NewAccesses); VectorMap[Load] = NewLoad; } @@ -768,9 +763,9 @@ void VectorBlockGenerator::copyBinaryInst(ScopStmt &Stmt, VectorMap[Inst] = NewInst; } -void VectorBlockGenerator::copyStore(ScopStmt &Stmt, const StoreInst *Store, - ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps) { +void VectorBlockGenerator::copyStore( + ScopStmt &Stmt, const StoreInst *Store, ValueMapT &VectorMap, + VectorValueMapT &ScalarMaps, __isl_keep isl_id_to_ast_expr *NewAccesses) { const MemoryAccess &Access = Stmt.getAccessFor(Store); const Value *Pointer = Store->getPointerOperand(); @@ -783,8 +778,9 @@ void VectorBlockGenerator::copyStore(ScopStmt &Stmt, const StoreInst *Store, if (Access.isStrideOne(isl_map_copy(Schedule))) { Type *VectorPtrType = getVectorPtrTy(Pointer, getVectorWidth()); - Value *NewPointer = generateLocationAccessed( - Stmt, Store, Pointer, ScalarMaps[0], GlobalMaps[0], VLTS[0]); + Value *NewPointer = + generateLocationAccessed(Stmt, Store, Pointer, ScalarMaps[0], + GlobalMaps[0], VLTS[0], NewAccesses); Value *VectorPtr = Builder.CreateBitCast(NewPointer, VectorPtrType, "vector_ptr"); @@ -795,8 +791,9 @@ void VectorBlockGenerator::copyStore(ScopStmt &Stmt, const StoreInst *Store, } else { for (unsigned i = 0; i < ScalarMaps.size(); i++) { Value *Scalar = Builder.CreateExtractElement(Vector, Builder.getInt32(i)); - Value *NewPointer = generateLocationAccessed( - Stmt, Store, Pointer, ScalarMaps[i], GlobalMaps[i], VLTS[i]); + Value *NewPointer = + generateLocationAccessed(Stmt, Store, Pointer, ScalarMaps[i], + GlobalMaps[i], VLTS[i], NewAccesses); Builder.CreateStore(Scalar, NewPointer); } } @@ -842,10 +839,9 @@ bool VectorBlockGenerator::extractScalarValues(const Instruction *Inst, return HasVectorOperand; } -void VectorBlockGenerator::copyInstScalarized(ScopStmt &Stmt, - const Instruction *Inst, - ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps) { +void VectorBlockGenerator::copyInstScalarized( + ScopStmt &Stmt, const Instruction *Inst, ValueMapT &VectorMap, + VectorValueMapT &ScalarMaps, __isl_keep isl_id_to_ast_expr *NewAccesses) { bool HasVectorOperand; int VectorWidth = getVectorWidth(); @@ -853,7 +849,8 @@ void VectorBlockGenerator::copyInstScalarized(ScopStmt &Stmt, for (int VectorLane = 0; VectorLane < getVectorWidth(); VectorLane++) BlockGenerator::copyInstruction(Stmt, Inst, ScalarMaps[VectorLane], - GlobalMaps[VectorLane], VLTS[VectorLane]); + GlobalMaps[VectorLane], VLTS[VectorLane], + NewAccesses); if (!VectorType::isValidElementType(Inst->getType()) || !HasVectorOperand) return; @@ -871,10 +868,9 @@ void VectorBlockGenerator::copyInstScalarized(ScopStmt &Stmt, int VectorBlockGenerator::getVectorWidth() { return GlobalMaps.size(); } -void VectorBlockGenerator::copyInstruction(ScopStmt &Stmt, - const Instruction *Inst, - ValueMapT &VectorMap, - VectorValueMapT &ScalarMaps) { +void VectorBlockGenerator::copyInstruction( + ScopStmt &Stmt, const Instruction *Inst, ValueMapT &VectorMap, + VectorValueMapT &ScalarMaps, __isl_keep isl_id_to_ast_expr *NewAccesses) { // Terminator instructions control the control flow. They are explicitly // expressed in the clast and do not need to be copied. if (Inst->isTerminator()) @@ -884,13 +880,13 @@ void VectorBlockGenerator::copyInstruction(ScopStmt &Stmt, return; if (const LoadInst *Load = dyn_cast<LoadInst>(Inst)) { - generateLoad(Stmt, Load, VectorMap, ScalarMaps); + generateLoad(Stmt, Load, VectorMap, ScalarMaps, NewAccesses); return; } if (hasVectorOperands(Inst, VectorMap)) { if (const StoreInst *Store = dyn_cast<StoreInst>(Inst)) { - copyStore(Stmt, Store, VectorMap, ScalarMaps); + copyStore(Stmt, Store, VectorMap, ScalarMaps, NewAccesses); return; } @@ -908,10 +904,11 @@ void VectorBlockGenerator::copyInstruction(ScopStmt &Stmt, // generate vector code. } - copyInstScalarized(Stmt, Inst, VectorMap, ScalarMaps); + copyInstScalarized(Stmt, Inst, VectorMap, ScalarMaps, NewAccesses); } -void VectorBlockGenerator::copyStmt(ScopStmt &Stmt) { +void VectorBlockGenerator::copyStmt( + ScopStmt &Stmt, __isl_keep isl_id_to_ast_expr *NewAccesses) { assert(Stmt.isBlockStmt() && "TODO: Only block statements can be copied by " "the vector block generator"); @@ -939,7 +936,7 @@ void VectorBlockGenerator::copyStmt(ScopStmt &Stmt) { ValueMapT VectorBlockMap; for (Instruction &Inst : *BB) - copyInstruction(Stmt, &Inst, VectorBlockMap, ScalarBlockMap); + copyInstruction(Stmt, &Inst, VectorBlockMap, ScalarBlockMap, NewAccesses); } BasicBlock *RegionGenerator::repairDominance(BasicBlock *BB, @@ -955,7 +952,8 @@ BasicBlock *RegionGenerator::repairDominance(BasicBlock *BB, } void RegionGenerator::copyStmt(ScopStmt &Stmt, ValueMapT &GlobalMap, - LoopToScevMapT <S) { + LoopToScevMapT <S, + isl_id_to_ast_expr *IdToAstExp) { assert(Stmt.isRegionStmt() && "Only region statements can be copied by the region generator"); @@ -1002,7 +1000,7 @@ void RegionGenerator::copyStmt(ScopStmt &Stmt, ValueMapT &GlobalMap, RegionMap = RegionMaps[BBCopyIDom]; // Copy the block with the BlockGenerator. - copyBB(Stmt, BB, BBCopy, RegionMap, GlobalMap, LTS); + copyBB(Stmt, BB, BBCopy, RegionMap, GlobalMap, LTS, IdToAstExp); // In order to remap PHI nodes we store also basic block mappings. BlockMap[BB] = BBCopy; diff --git a/polly/lib/CodeGen/IslNodeBuilder.cpp b/polly/lib/CodeGen/IslNodeBuilder.cpp index b6d4d87ae6d..3590ab5542a 100644 --- a/polly/lib/CodeGen/IslNodeBuilder.cpp +++ b/polly/lib/CodeGen/IslNodeBuilder.cpp @@ -286,7 +286,6 @@ void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User, isl_id *Id = isl_ast_expr_get_id(StmtExpr); isl_ast_expr_free(StmtExpr); ScopStmt *Stmt = (ScopStmt *)isl_id_get_user(Id); - Stmt->setAstBuild(IslAstInfo::getBuild(User)); VectorValueMapT VectorMap(IVS.size()); std::vector<LoopToScevMapT> VLTS(IVS.size()); @@ -294,9 +293,11 @@ void IslNodeBuilder::createUserVector(__isl_take isl_ast_node *User, Schedule = isl_union_map_intersect_domain(Schedule, Domain); isl_map *S = isl_map_from_union_map(Schedule); + auto *NewAccesses = createNewAccesses(Stmt, IslAstInfo::getBuild(User)); createSubstitutionsVector(Expr, Stmt, VectorMap, VLTS, IVS, IteratorID); - VectorBlockGenerator::generate(BlockGen, *Stmt, VectorMap, VLTS, S); - + VectorBlockGenerator::generate(BlockGen, *Stmt, VectorMap, VLTS, S, + NewAccesses); + isl_id_to_ast_expr_free(NewAccesses); isl_map_free(S); isl_id_free(Id); isl_ast_node_free(User); @@ -640,6 +641,25 @@ void IslNodeBuilder::createIf(__isl_take isl_ast_node *If) { isl_ast_node_free(If); } +__isl_give isl_id_to_ast_expr * +IslNodeBuilder::createNewAccesses(ScopStmt *Stmt, + __isl_keep isl_ast_build *Build) { + isl_id_to_ast_expr *NewAccesses = + isl_id_to_ast_expr_alloc(isl_ast_build_get_ctx(Build), 0); + for (auto *MA : *Stmt) { + if (!MA->hasNewAccessRelation()) + continue; + + auto Schedule = isl_ast_build_get_schedule(Build); + auto PWAccRel = MA->applyScheduleToAccessRelation(Schedule); + + auto AccessExpr = isl_ast_build_access_from_pw_multi_aff(Build, PWAccRel); + NewAccesses = isl_id_to_ast_expr_set(NewAccesses, MA->getId(), AccessExpr); + } + + return NewAccesses; +} + void IslNodeBuilder::createSubstitutions(isl_ast_expr *Expr, ScopStmt *Stmt, ValueMapT &VMap, LoopToScevMapT <S) { assert(isl_ast_expr_get_type(Expr) == isl_ast_expr_op && @@ -697,14 +717,15 @@ void IslNodeBuilder::createUser(__isl_take isl_ast_node *User) { LTS.insert(OutsideLoopIterations.begin(), OutsideLoopIterations.end()); Stmt = (ScopStmt *)isl_id_get_user(Id); - Stmt->setAstBuild(IslAstInfo::getBuild(User)); - + auto *NewAccesses = createNewAccesses(Stmt, IslAstInfo::getBuild(User)); createSubstitutions(Expr, Stmt, VMap, LTS); + if (Stmt->isBlockStmt()) - BlockGen.copyStmt(*Stmt, VMap, LTS); + BlockGen.copyStmt(*Stmt, VMap, LTS, NewAccesses); else - RegionGen.copyStmt(*Stmt, VMap, LTS); + RegionGen.copyStmt(*Stmt, VMap, LTS, NewAccesses); + isl_id_to_ast_expr_free(NewAccesses); isl_ast_node_free(User); isl_id_free(Id); } |
