diff options
Diffstat (limited to 'clang/lib/CodeGen/CGOpenMPRuntime.cpp')
| -rw-r--r-- | clang/lib/CodeGen/CGOpenMPRuntime.cpp | 587 |
1 files changed, 300 insertions, 287 deletions
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 75103db2667..810a2db3a5b 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -252,7 +252,7 @@ private: StringRef HelperName; }; -static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) { +static void EmptyCodeGen(CodeGenFunction &) { llvm_unreachable("No codegen for expressions"); } /// \brief API for generation of expressions captured in a innermost OpenMP @@ -564,33 +564,8 @@ enum OpenMPRTLFunction { OMPRTL__tgt_unregister_lib, }; -/// A basic class for pre|post-action for advanced codegen sequence for OpenMP -/// region. -class CleanupTy final : public EHScopeStack::Cleanup { - PrePostActionTy *Action; - -public: - explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {} - void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { - if (!CGF.HaveInsertPoint()) - return; - Action->Exit(CGF); - } -}; - } // anonymous namespace -void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { - CodeGenFunction::RunCleanupsScope Scope(CGF); - if (PrePostAction) { - CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); - Callback(CodeGen, CGF, *PrePostAction); - } else { - PrePostActionTy Action; - Callback(CodeGen, CGF, Action); - } -} - LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { return CGF.EmitLoadOfPointerLValue( CGF.GetAddrOfLocalVar(getThreadIDVariable()), @@ -606,7 +581,10 @@ void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { // The point of exit cannot be a branch out of the structured block. // longjmp() and throw() must not violate the entry/exit criteria. CGF.EHStack.pushTerminate(); - CodeGen(CGF); + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CodeGen(CGF); + } CGF.EHStack.popTerminate(); } @@ -1666,10 +1644,12 @@ static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, // the condition and the dead arm of the if/else. bool CondConstant; if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { - if (CondConstant) + CodeGenFunction::RunCleanupsScope Scope(CGF); + if (CondConstant) { ThenGen(CGF); - else + } else { ElseGen(CGF); + } return; } @@ -1682,16 +1662,26 @@ static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, // Emit the 'then' code. CGF.EmitBlock(ThenBlock); - ThenGen(CGF); + { + CodeGenFunction::RunCleanupsScope ThenScope(CGF); + ThenGen(CGF); + } CGF.EmitBranch(ContBlock); // Emit the 'else' code if present. - // There is no need to emit line number for unconditional branch. - (void)ApplyDebugLocation::CreateEmpty(CGF); - CGF.EmitBlock(ElseBlock); - ElseGen(CGF); - // There is no need to emit line number for unconditional branch. - (void)ApplyDebugLocation::CreateEmpty(CGF); - CGF.EmitBranch(ContBlock); + { + // There is no need to emit line number for unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(CGF); + CGF.EmitBlock(ElseBlock); + } + { + CodeGenFunction::RunCleanupsScope ThenScope(CGF); + ElseGen(CGF); + } + { + // There is no need to emit line number for unconditional branch. + auto NL = ApplyDebugLocation::CreateEmpty(CGF); + CGF.EmitBranch(ContBlock); + } // Emit the continuation block for code after the if. CGF.EmitBlock(ContBlock, /*IsFinished=*/true); } @@ -1703,8 +1693,8 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, if (!CGF.HaveInsertPoint()) return; auto *RTLoc = emitUpdateLocation(CGF, Loc); - RegionCodeGenTy ThenGen = [this, OutlinedFn, CapturedVars, - RTLoc](CodeGenFunction &CGF, PrePostActionTy &) { + auto &&ThenGen = [this, OutlinedFn, CapturedVars, + RTLoc](CodeGenFunction &CGF) { // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn); llvm::Value *Args[] = { RTLoc, @@ -1717,8 +1707,8 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, auto RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_call); CGF.EmitRuntimeCall(RTLFn, RealArgs); }; - RegionCodeGenTy ElseGen = [this, OutlinedFn, CapturedVars, RTLoc, - Loc](CodeGenFunction &CGF, PrePostActionTy &) { + auto &&ElseGen = [this, OutlinedFn, CapturedVars, RTLoc, + Loc](CodeGenFunction &CGF) { auto ThreadID = getThreadID(CGF, Loc); // Build calls: // __kmpc_serialized_parallel(&Loc, GTid); @@ -1743,10 +1733,12 @@ void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, CGF.EmitRuntimeCall( createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel), EndArgs); }; - if (IfCond) + if (IfCond) { emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); - else + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); ThenGen(CGF); + } } // If we're inside an (outlined) parallel region, use the region info's @@ -1798,39 +1790,21 @@ llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) { } namespace { -/// Common pre(post)-action for different OpenMP constructs. -class CommonActionTy final : public PrePostActionTy { - llvm::Value *EnterCallee; - ArrayRef<llvm::Value *> EnterArgs; - llvm::Value *ExitCallee; - ArrayRef<llvm::Value *> ExitArgs; - bool Conditional; - llvm::BasicBlock *ContBlock = nullptr; +template <size_t N> class CallEndCleanup final : public EHScopeStack::Cleanup { + llvm::Value *Callee; + llvm::Value *Args[N]; public: - CommonActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs, - llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs, - bool Conditional = false) - : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee), - ExitArgs(ExitArgs), Conditional(Conditional) {} - void Enter(CodeGenFunction &CGF) override { - llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); - if (Conditional) { - llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); - auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); - ContBlock = CGF.createBasicBlock("omp_if.end"); - // Generate the branch (If-stmt) - CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); - CGF.EmitBlock(ThenBlock); - } - } - void Done(CodeGenFunction &CGF) { - // Emit the rest of blocks/branches - CGF.EmitBranch(ContBlock); - CGF.EmitBlock(ContBlock, true); + CallEndCleanup(llvm::Value *Callee, ArrayRef<llvm::Value *> CleanupArgs) + : Callee(Callee) { + assert(CleanupArgs.size() == N); + std::copy(CleanupArgs.begin(), CleanupArgs.end(), std::begin(Args)); } - void Exit(CodeGenFunction &CGF) override { - CGF.EmitRuntimeCall(ExitCallee, ExitArgs); + + void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { + if (!CGF.HaveInsertPoint()) + return; + CGF.EmitRuntimeCall(Callee, Args); } }; } // anonymous namespace @@ -1845,22 +1819,45 @@ void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, // Prepare arguments and build a call to __kmpc_critical if (!CGF.HaveInsertPoint()) return; + CodeGenFunction::RunCleanupsScope Scope(CGF); llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), getCriticalRegionLock(CriticalName)}; - llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), - std::end(Args)); if (Hint) { - EnterArgs.push_back(CGF.Builder.CreateIntCast( - CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false)); - } - CommonActionTy Action( - createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint - : OMPRTL__kmpc_critical), - EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args); - CriticalOpGen.setAction(Action); + llvm::SmallVector<llvm::Value *, 8> ArgsWithHint(std::begin(Args), + std::end(Args)); + auto *HintVal = CGF.EmitScalarExpr(Hint); + ArgsWithHint.push_back( + CGF.Builder.CreateIntCast(HintVal, CGM.IntPtrTy, /*isSigned=*/false)); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical_with_hint), + ArgsWithHint); + } else + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_critical), Args); + // Build a call to __kmpc_end_critical + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_critical), + llvm::makeArrayRef(Args)); emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); } +static void emitIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond, + OpenMPDirectiveKind Kind, SourceLocation Loc, + const RegionCodeGenTy &BodyOpGen) { + llvm::Value *CallBool = CGF.EmitScalarConversion( + IfCond, + CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true), + CGF.getContext().BoolTy, Loc); + + auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); + auto *ContBlock = CGF.createBasicBlock("omp_if.end"); + // Generate the branch (If-stmt) + CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); + CGF.EmitBlock(ThenBlock); + CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, Kind, BodyOpGen); + // Emit the rest of bblocks/branches + CGF.EmitBranch(ContBlock); + CGF.EmitBlock(ContBlock, true); +} + void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc) { @@ -1872,12 +1869,18 @@ void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, // } // Prepare arguments and build a call to __kmpc_master llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; - CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args, - createRuntimeFunction(OMPRTL__kmpc_end_master), Args, - /*Conditional=*/true); - MasterOpGen.setAction(Action); - emitInlinedDirective(CGF, OMPD_master, MasterOpGen); - Action.Done(CGF); + auto *IsMaster = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_master), Args); + typedef CallEndCleanup<std::extent<decltype(Args)>::value> + MasterCallEndCleanup; + emitIfStmt( + CGF, IsMaster, OMPD_master, Loc, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<MasterCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_master), + llvm::makeArrayRef(Args)); + MasterOpGen(CGF); + }); } void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, @@ -1900,12 +1903,16 @@ void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, // TaskgroupOpGen(); // __kmpc_end_taskgroup(ident_t *, gtid); // Prepare arguments and build a call to __kmpc_taskgroup - llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; - CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args, - createRuntimeFunction(OMPRTL__kmpc_end_taskgroup), - Args); - TaskgroupOpGen.setAction(Action); - emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args); + // Build a call to __kmpc_end_taskgroup + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_taskgroup), + llvm::makeArrayRef(Args)); + emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); + } } /// Given an array of pointers to variables, project the address of a @@ -2001,16 +2008,22 @@ void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, } // Prepare arguments and build a call to __kmpc_single llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; - CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args, - createRuntimeFunction(OMPRTL__kmpc_end_single), Args, - /*Conditional=*/true); - SingleOpGen.setAction(Action); - emitInlinedDirective(CGF, OMPD_single, SingleOpGen); - if (DidIt.isValid()) { - // did_it = 1; - CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); - } - Action.Done(CGF); + auto *IsSingle = + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_single), Args); + typedef CallEndCleanup<std::extent<decltype(Args)>::value> + SingleCallEndCleanup; + emitIfStmt( + CGF, IsSingle, OMPD_single, Loc, [&](CodeGenFunction &CGF) -> void { + CodeGenFunction::RunCleanupsScope Scope(CGF); + CGF.EHStack.pushCleanup<SingleCallEndCleanup>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_single), + llvm::makeArrayRef(Args)); + SingleOpGen(CGF); + if (DidIt.isValid()) { + // did_it = 1; + CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); + } + }); // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, // <copy_func>, did_it); if (DidIt.isValid()) { @@ -2060,14 +2073,14 @@ void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, // OrderedOpGen(); // __kmpc_end_ordered(ident_t *, gtid); // Prepare arguments and build a call to __kmpc_ordered + CodeGenFunction::RunCleanupsScope Scope(CGF); if (IsThreads) { llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; - CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args, - createRuntimeFunction(OMPRTL__kmpc_end_ordered), - Args); - OrderedOpGen.setAction(Action); - emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); - return; + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_ordered), Args); + // Build a call to __kmpc_end_ordered + CGF.EHStack.pushCleanup<CallEndCleanup<std::extent<decltype(Args)>::value>>( + NormalAndEHCleanup, createRuntimeFunction(OMPRTL__kmpc_end_ordered), + llvm::makeArrayRef(Args)); } emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); } @@ -2583,14 +2596,12 @@ CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() { IdentInfo, C.CharTy); auto *UnRegFn = createOffloadingBinaryDescriptorFunction( - CGM, ".omp_offloading.descriptor_unreg", - [&](CodeGenFunction &CGF, PrePostActionTy &) { + CGM, ".omp_offloading.descriptor_unreg", [&](CodeGenFunction &CGF) { CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_unregister_lib), Desc); }); auto *RegFn = createOffloadingBinaryDescriptorFunction( - CGM, ".omp_offloading.descriptor_reg", - [&](CodeGenFunction &CGF, PrePostActionTy &) { + CGM, ".omp_offloading.descriptor_reg", [&](CodeGenFunction &CGF) { CGF.EmitCallOrInvoke(createRuntimeFunction(OMPRTL__tgt_register_lib), Desc); CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc); @@ -3458,10 +3469,9 @@ void CGOpenMPRuntime::emitTaskCall( DepTaskArgs[5] = CGF.Builder.getInt32(0); DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); } - RegionCodeGenTy ThenCodeGen = [this, NumDependencies, &TaskArgs, - &DepTaskArgs](CodeGenFunction &CGF, - PrePostActionTy &) { - // TODO: add check for untied tasks. + auto &&ThenCodeGen = [this, NumDependencies, + &TaskArgs, &DepTaskArgs](CodeGenFunction &CGF) { + // TODO: add check for untied tasks. if (NumDependencies) { CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs); @@ -3470,6 +3480,8 @@ void CGOpenMPRuntime::emitTaskCall( TaskArgs); } }; + typedef CallEndCleanup<std::extent<decltype(TaskArgs)>::value> + IfCallEndCleanup; llvm::Value *DepWaitTaskArgs[6]; if (NumDependencies) { @@ -3481,8 +3493,7 @@ void CGOpenMPRuntime::emitTaskCall( DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); } auto &&ElseCodeGen = [this, &TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry, - NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF, - PrePostActionTy &) { + NumDependencies, &DepWaitTaskArgs](CodeGenFunction &CGF) { CodeGenFunction::RunCleanupsScope LocalScope(CGF); // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid, // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 @@ -3491,29 +3502,28 @@ void CGOpenMPRuntime::emitTaskCall( if (NumDependencies) CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps), DepWaitTaskArgs); - // Call proxy_task_entry(gtid, new_task); - RegionCodeGenTy CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy]( - CodeGenFunction &CGF, PrePostActionTy &Action) { - Action.Enter(CGF); - llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy}; - CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs); - }; - // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, // kmp_task_t *new_task); + CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), + TaskArgs); // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, // kmp_task_t *new_task); - CommonActionTy Action( - createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs, - createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs); - CodeGen.setAction(Action); - CodeGen(CGF); + CGF.EHStack.pushCleanup<IfCallEndCleanup>( + NormalAndEHCleanup, + createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), + llvm::makeArrayRef(TaskArgs)); + + // Call proxy_task_entry(gtid, new_task); + llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy}; + CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs); }; - if (IfCond) + if (IfCond) { emitOMPIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); - else + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); ThenCodeGen(CGF); + } } /// \brief Emit reduction operation for each element of array (required for @@ -3704,25 +3714,6 @@ static llvm::Value *emitReductionFunction(CodeGenModule &CGM, return Fn; } -static void emitSingleReductionCombiner(CodeGenFunction &CGF, - const Expr *ReductionOp, - const Expr *PrivateRef, - const DeclRefExpr *LHS, - const DeclRefExpr *RHS) { - if (PrivateRef->getType()->isArrayType()) { - // Emit reduction for array section. - auto *LHSVar = cast<VarDecl>(LHS->getDecl()); - auto *RHSVar = cast<VarDecl>(RHS->getDecl()); - EmitOMPAggregateReduction( - CGF, PrivateRef->getType(), LHSVar, RHSVar, - [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { - emitReductionCombiner(CGF, ReductionOp); - }); - } else - // Emit reduction for array subscript or single variable. - emitReductionCombiner(CGF, ReductionOp); -} - void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, ArrayRef<const Expr *> LHSExprs, @@ -3774,8 +3765,15 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, auto ILHS = LHSExprs.begin(); auto IRHS = RHSExprs.begin(); for (auto *E : ReductionOps) { - emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), - cast<DeclRefExpr>(*IRHS)); + if ((*IPriv)->getType()->isArrayType()) { + auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction( + CGF, (*IPriv)->getType(), LHSVar, RHSVar, + [=](CodeGenFunction &CGF, const Expr *, const Expr *, + const Expr *) { emitReductionCombiner(CGF, E); }); + } else + emitReductionCombiner(CGF, E); ++IPriv; ++ILHS; ++IRHS; @@ -3865,32 +3863,40 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); CGF.EmitBlock(Case1BB); - // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); - llvm::Value *EndArgs[] = { - IdentTLoc, // ident_t *<loc> - ThreadId, // i32 <gtid> - Lock // kmp_critical_name *&<lock> - }; - RegionCodeGenTy CodeGen = [&Privates, &LHSExprs, &RHSExprs, &ReductionOps]( - CodeGenFunction &CGF, PrePostActionTy &Action) { + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); + llvm::Value *EndArgs[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + Lock // kmp_critical_name *&<lock> + }; + CGF.EHStack + .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>( + NormalAndEHCleanup, + createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait + : OMPRTL__kmpc_end_reduce), + llvm::makeArrayRef(EndArgs)); auto IPriv = Privates.begin(); auto ILHS = LHSExprs.begin(); auto IRHS = RHSExprs.begin(); for (auto *E : ReductionOps) { - emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), - cast<DeclRefExpr>(*IRHS)); + if ((*IPriv)->getType()->isArrayType()) { + // Emit reduction for array section. + auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction( + CGF, (*IPriv)->getType(), LHSVar, RHSVar, + [=](CodeGenFunction &CGF, const Expr *, const Expr *, + const Expr *) { emitReductionCombiner(CGF, E); }); + } else + // Emit reduction for array subscript or single variable. + emitReductionCombiner(CGF, E); ++IPriv; ++ILHS; ++IRHS; } - }; - CommonActionTy Action( - nullptr, llvm::None, - createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait - : OMPRTL__kmpc_end_reduce), - EndArgs); - CodeGen.setAction(Action); - CodeGen(CGF); + } CGF.EmitBranch(DefaultBB); @@ -3903,111 +3909,106 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); CGF.EmitBlock(Case2BB); - RegionCodeGenTy AtomicCodeGen = [this, Loc, &Privates, &LHSExprs, &RHSExprs, - &ReductionOps](CodeGenFunction &CGF, - PrePostActionTy &Action) { + { + CodeGenFunction::RunCleanupsScope Scope(CGF); + if (!WithNowait) { + // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>); + llvm::Value *EndArgs[] = { + IdentTLoc, // ident_t *<loc> + ThreadId, // i32 <gtid> + Lock // kmp_critical_name *&<lock> + }; + CGF.EHStack + .pushCleanup<CallEndCleanup<std::extent<decltype(EndArgs)>::value>>( + NormalAndEHCleanup, + createRuntimeFunction(OMPRTL__kmpc_end_reduce), + llvm::makeArrayRef(EndArgs)); + } auto ILHS = LHSExprs.begin(); auto IRHS = RHSExprs.begin(); auto IPriv = Privates.begin(); for (auto *E : ReductionOps) { - const Expr *XExpr = nullptr; - const Expr *EExpr = nullptr; - const Expr *UpExpr = nullptr; - BinaryOperatorKind BO = BO_Comma; - if (auto *BO = dyn_cast<BinaryOperator>(E)) { - if (BO->getOpcode() == BO_Assign) { - XExpr = BO->getLHS(); - UpExpr = BO->getRHS(); + const Expr *XExpr = nullptr; + const Expr *EExpr = nullptr; + const Expr *UpExpr = nullptr; + BinaryOperatorKind BO = BO_Comma; + if (auto *BO = dyn_cast<BinaryOperator>(E)) { + if (BO->getOpcode() == BO_Assign) { + XExpr = BO->getLHS(); + UpExpr = BO->getRHS(); + } } - } - // Try to emit update expression as a simple atomic. - auto *RHSExpr = UpExpr; - if (RHSExpr) { - // Analyze RHS part of the whole expression. - if (auto *ACO = dyn_cast<AbstractConditionalOperator>( - RHSExpr->IgnoreParenImpCasts())) { - // If this is a conditional operator, analyze its condition for - // min/max reduction operator. - RHSExpr = ACO->getCond(); + // Try to emit update expression as a simple atomic. + auto *RHSExpr = UpExpr; + if (RHSExpr) { + // Analyze RHS part of the whole expression. + if (auto *ACO = dyn_cast<AbstractConditionalOperator>( + RHSExpr->IgnoreParenImpCasts())) { + // If this is a conditional operator, analyze its condition for + // min/max reduction operator. + RHSExpr = ACO->getCond(); + } + if (auto *BORHS = + dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { + EExpr = BORHS->getRHS(); + BO = BORHS->getOpcode(); + } } - if (auto *BORHS = - dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { - EExpr = BORHS->getRHS(); - BO = BORHS->getOpcode(); + if (XExpr) { + auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto &&AtomicRedGen = [this, BO, VD, IPriv, + Loc](CodeGenFunction &CGF, const Expr *XExpr, + const Expr *EExpr, const Expr *UpExpr) { + LValue X = CGF.EmitLValue(XExpr); + RValue E; + if (EExpr) + E = CGF.EmitAnyExpr(EExpr); + CGF.EmitOMPAtomicSimpleUpdateExpr( + X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc, + [&CGF, UpExpr, VD, IPriv, Loc](RValue XRValue) { + CodeGenFunction::OMPPrivateScope PrivateScope(CGF); + PrivateScope.addPrivate( + VD, [&CGF, VD, XRValue, Loc]() -> Address { + Address LHSTemp = CGF.CreateMemTemp(VD->getType()); + CGF.emitOMPSimpleStore( + CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, + VD->getType().getNonReferenceType(), Loc); + return LHSTemp; + }); + (void)PrivateScope.Privatize(); + return CGF.EmitAnyExpr(UpExpr); + }); + }; + if ((*IPriv)->getType()->isArrayType()) { + // Emit atomic reduction for array section. + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, + AtomicRedGen, XExpr, EExpr, UpExpr); + } else + // Emit atomic reduction for array subscript or single variable. + AtomicRedGen(CGF, XExpr, EExpr, UpExpr); + } else { + // Emit as a critical region. + auto &&CritRedGen = [this, E, Loc](CodeGenFunction &CGF, const Expr *, + const Expr *, const Expr *) { + emitCriticalRegion( + CGF, ".atomic_reduction", + [=](CodeGenFunction &CGF) { emitReductionCombiner(CGF, E); }, + Loc); + }; + if ((*IPriv)->getType()->isArrayType()) { + auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); + auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); + EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, + CritRedGen); + } else + CritRedGen(CGF, nullptr, nullptr, nullptr); } - } - if (XExpr) { - auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); - auto &&AtomicRedGen = [this, BO, VD, IPriv, - Loc](CodeGenFunction &CGF, const Expr *XExpr, - const Expr *EExpr, const Expr *UpExpr) { - LValue X = CGF.EmitLValue(XExpr); - RValue E; - if (EExpr) - E = CGF.EmitAnyExpr(EExpr); - CGF.EmitOMPAtomicSimpleUpdateExpr( - X, E, BO, /*IsXLHSInRHSPart=*/true, llvm::Monotonic, Loc, - [&CGF, UpExpr, VD, IPriv, Loc](RValue XRValue) { - CodeGenFunction::OMPPrivateScope PrivateScope(CGF); - PrivateScope.addPrivate( - VD, [&CGF, VD, XRValue, Loc]() -> Address { - Address LHSTemp = CGF.CreateMemTemp(VD->getType()); - CGF.emitOMPSimpleStore( - CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, - VD->getType().getNonReferenceType(), Loc); - return LHSTemp; - }); - (void)PrivateScope.Privatize(); - return CGF.EmitAnyExpr(UpExpr); - }); - }; - if ((*IPriv)->getType()->isArrayType()) { - // Emit atomic reduction for array section. - auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); - EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, - AtomicRedGen, XExpr, EExpr, UpExpr); - } else - // Emit atomic reduction for array subscript or single variable. - AtomicRedGen(CGF, XExpr, EExpr, UpExpr); - } else { - // Emit as a critical region. - auto &&CritRedGen = [this, E, Loc](CodeGenFunction &CGF, const Expr *, - const Expr *, const Expr *) { - emitCriticalRegion( - CGF, ".atomic_reduction", - [=](CodeGenFunction &CGF, PrePostActionTy &Action) { - Action.Enter(CGF); - emitReductionCombiner(CGF, E); - }, - Loc); - }; - if ((*IPriv)->getType()->isArrayType()) { - auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); - auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); - EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, - CritRedGen); - } else - CritRedGen(CGF, nullptr, nullptr, nullptr); - } ++ILHS; ++IRHS; ++IPriv; } - }; - if (!WithNowait) { - // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>); - llvm::Value *EndArgs[] = { - IdentTLoc, // ident_t *<loc> - ThreadId, // i32 <gtid> - Lock // kmp_critical_name *&<lock> - }; - CommonActionTy Action(nullptr, llvm::None, - createRuntimeFunction(OMPRTL__kmpc_end_reduce), - EndArgs); - AtomicCodeGen.setAction(Action); - AtomicCodeGen(CGF); - } else - AtomicCodeGen(CGF); + } CGF.EmitBranch(DefaultBB); CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); @@ -4104,8 +4105,8 @@ void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, // kmp_int32 cncl_kind); if (auto *OMPRegionInfo = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { - RegionCodeGenTy ThenGen = [this, Loc, CancelRegion, OMPRegionInfo]( - CodeGenFunction &CGF, PrePostActionTy &) { + auto &&ThenGen = [this, Loc, CancelRegion, + OMPRegionInfo](CodeGenFunction &CGF) { llvm::Value *Args[] = { emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; @@ -4130,8 +4131,7 @@ void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, CGF.EmitBlock(ContBB, /*IsFinished=*/true); }; if (IfCond) - emitOMPIfClause(CGF, IfCond, ThenGen, - [](CodeGenFunction &, PrePostActionTy &) {}); + emitOMPIfClause(CGF, IfCond, ThenGen, [](CodeGenFunction &) {}); else ThenGen(CGF); } @@ -4167,9 +4167,21 @@ static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc, void CGOpenMPRuntime::emitTargetOutlinedFunction( const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, - bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { + bool IsOffloadEntry) { assert(!ParentName.empty() && "Invalid target region parent name!"); + const CapturedStmt &CS = *cast<CapturedStmt>(D.getAssociatedStmt()); + + // Emit target region as a standalone region. + auto &&CodeGen = [&CS, &D](CodeGenFunction &CGF) { + CodeGenFunction::OMPPrivateScope PrivateScope(CGF); + (void)CGF.EmitOMPFirstprivateClause(D, PrivateScope); + CGF.EmitOMPPrivateClause(D, PrivateScope); + (void)PrivateScope.Privatize(); + + CGF.EmitStmt(CS.getCapturedStmt()); + }; + emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry, CodeGen); } @@ -4459,10 +4471,9 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, OffloadError); // Fill up the pointer arrays and transfer execution to the device. - RegionCodeGenTy ThenGen = [this, &Ctx, &BasePointers, &Pointers, &Sizes, - &MapTypes, hasVLACaptures, Device, OutlinedFnID, - OffloadError, OffloadErrorQType, - &D](CodeGenFunction &CGF, PrePostActionTy &) { + auto &&ThenGen = [this, &Ctx, &BasePointers, &Pointers, &Sizes, &MapTypes, + hasVLACaptures, Device, OutlinedFnID, OffloadError, + OffloadErrorQType, &D](CodeGenFunction &CGF) { unsigned PointerNumVal = BasePointers.size(); llvm::Value *PointerNum = CGF.Builder.getInt32(PointerNumVal); llvm::Value *BasePointersArray; @@ -4635,8 +4646,8 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, }; // Notify that the host version must be executed. - RegionCodeGenTy ElseGen = [this, OffloadError, OffloadErrorQType]( - CodeGenFunction &CGF, PrePostActionTy &) { + auto &&ElseGen = [this, OffloadError, + OffloadErrorQType](CodeGenFunction &CGF) { CGF.EmitStoreOfScalar(llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/-1u), OffloadError); }; @@ -4646,12 +4657,16 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF, // regardless of the conditional in the if clause if, e.g., the user do not // specify target triples. if (OutlinedFnID) { - if (IfCond) + if (IfCond) { emitOMPIfClause(CGF, IfCond, ThenGen, ElseGen); - else + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); ThenGen(CGF); - } else + } + } else { + CodeGenFunction::RunCleanupsScope Scope(CGF); ElseGen(CGF); + } // Check the error code and execute the host version if required. auto OffloadFailedBlock = CGF.createBasicBlock("omp_offload.failed"); @@ -4693,10 +4708,8 @@ void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S, llvm::Function *Fn; llvm::Constant *Addr; - std::tie(Fn, Addr) = - CodeGenFunction::EmitOMPTargetDirectiveOutlinedFunction( - CGM, cast<OMPTargetDirective>(*E), ParentName, - /*isOffloadEntry=*/true); + emitTargetOutlinedFunction(*E, ParentName, Fn, Addr, + /*isOffloadEntry=*/true); assert(Fn && Addr && "Target region emission failed."); return; } |
