summaryrefslogtreecommitdiffstats
path: root/clang/lib/CodeGen
diff options
context:
space:
mode:
Diffstat (limited to 'clang/lib/CodeGen')
-rw-r--r--clang/lib/CodeGen/CGOpenMPRuntime.cpp58
-rw-r--r--clang/lib/CodeGen/CGOpenMPRuntime.h3
-rw-r--r--clang/lib/CodeGen/CGStmtOpenMP.cpp155
3 files changed, 184 insertions, 32 deletions
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 64af40bd282..5bf12e39c10 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -622,12 +622,12 @@ emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
auto &C = CGM.getContext();
QualType PtrTy = C.getPointerType(Ty).withRestrict();
FunctionArgList Args;
- ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
- /*Id=*/nullptr, PtrTy);
ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(),
/*Id=*/nullptr, PtrTy);
- Args.push_back(&OmpInParm);
+ ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(),
+ /*Id=*/nullptr, PtrTy);
Args.push_back(&OmpOutParm);
+ Args.push_back(&OmpInParm);
auto &FnInfo =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
@@ -635,6 +635,7 @@ emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty,
FnTy, llvm::GlobalValue::InternalLinkage,
IsCombiner ? ".omp_combiner." : ".omp_initializer.", &CGM.getModule());
CGM.SetInternalFunctionAttributes(/*D=*/nullptr, Fn, FnInfo);
+ Fn->addFnAttr(llvm::Attribute::AlwaysInline);
CodeGenFunction CGF(CGM);
// Map "T omp_in;" variable to "*omp_in_parm" value in all expressions.
// Map "T omp_out;" variable to "*omp_out_parm" value in all expressions.
@@ -688,6 +689,15 @@ void CGOpenMPRuntime::emitUserDefinedReduction(
}
}
+std::pair<llvm::Function *, llvm::Function *>
+CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) {
+ auto I = UDRMap.find(D);
+ if (I != UDRMap.end())
+ return I->second;
+ emitUserDefinedReduction(/*CGF=*/nullptr, D);
+ return UDRMap.lookup(D);
+}
+
// Layout information for ident_t.
static CharUnits getIdentAlign(CodeGenModule &CGM) {
return CGM.getPointerAlign();
@@ -3596,6 +3606,26 @@ static void EmitOMPAggregateReduction(
CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
}
+/// Emit reduction combiner. If the combiner is a simple expression emit it as
+/// is, otherwise consider it as combiner of UDR decl and emit it as a call of
+/// UDR combiner function.
+static void emitReductionCombiner(CodeGenFunction &CGF,
+ const Expr *ReductionOp) {
+ if (auto *CE = dyn_cast<CallExpr>(ReductionOp))
+ if (auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
+ if (auto *DRE =
+ dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
+ if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
+ std::pair<llvm::Function *, llvm::Function *> Reduction =
+ CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
+ RValue Func = RValue::get(Reduction.first);
+ CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
+ CGF.EmitIgnoredExpr(ReductionOp);
+ return;
+ }
+ CGF.EmitIgnoredExpr(ReductionOp);
+}
+
static llvm::Value *emitReductionFunction(CodeGenModule &CGM,
llvm::Type *ArgsType,
ArrayRef<const Expr *> Privates,
@@ -3667,13 +3697,14 @@ static llvm::Value *emitReductionFunction(CodeGenModule &CGM,
// Emit reduction for array section.
auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
- EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar,
- [=](CodeGenFunction &CGF, const Expr *,
- const Expr *,
- const Expr *) { CGF.EmitIgnoredExpr(E); });
+ EmitOMPAggregateReduction(
+ CGF, (*IPriv)->getType(), LHSVar, RHSVar,
+ [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) {
+ emitReductionCombiner(CGF, E);
+ });
} else
// Emit reduction for array subscript or single variable.
- CGF.EmitIgnoredExpr(E);
+ emitReductionCombiner(CGF, E);
++IPriv;
++ILHS;
++IRHS;
@@ -3740,9 +3771,9 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
EmitOMPAggregateReduction(
CGF, (*IPriv)->getType(), LHSVar, RHSVar,
[=](CodeGenFunction &CGF, const Expr *, const Expr *,
- const Expr *) { CGF.EmitIgnoredExpr(E); });
+ const Expr *) { emitReductionCombiner(CGF, E); });
} else
- CGF.EmitIgnoredExpr(E);
+ emitReductionCombiner(CGF, E);
++IPriv;
++ILHS;
++IRHS;
@@ -3857,10 +3888,10 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
EmitOMPAggregateReduction(
CGF, (*IPriv)->getType(), LHSVar, RHSVar,
[=](CodeGenFunction &CGF, const Expr *, const Expr *,
- const Expr *) { CGF.EmitIgnoredExpr(E); });
+ const Expr *) { emitReductionCombiner(CGF, E); });
} else
// Emit reduction for array subscript or single variable.
- CGF.EmitIgnoredExpr(E);
+ emitReductionCombiner(CGF, E);
++IPriv;
++ILHS;
++IRHS;
@@ -3962,7 +3993,8 @@ void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc,
const Expr *, const Expr *) {
emitCriticalRegion(
CGF, ".atomic_reduction",
- [E](CodeGenFunction &CGF) { CGF.EmitIgnoredExpr(E); }, Loc);
+ [=](CodeGenFunction &CGF) { emitReductionCombiner(CGF, E); },
+ Loc);
};
if ((*IPriv)->getType()->isArrayType()) {
auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 97f08fc1e43..e9702170865 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -383,6 +383,9 @@ public:
/// Emit code for the specified user defined reduction construct.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
const OMPDeclareReductionDecl *D);
+ /// Get combiner/initializer for the specified user-defined reduction, if any.
+ virtual std::pair<llvm::Function *, llvm::Function *>
+ getUserDefinedReduction(const OMPDeclareReductionDecl *D);
/// \brief Emits outlined function for the specified OpenMP parallel directive
/// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID,
/// kmp_int32 BoundID, struct context_vars*).
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index eb62428eb38..30385cd8b92 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -19,6 +19,7 @@
#include "clang/AST/Stmt.h"
#include "clang/AST/StmtOpenMP.h"
#include "clang/AST/DeclOpenMP.h"
+#include "llvm/IR/CallSite.h"
using namespace clang;
using namespace CodeGen;
@@ -310,12 +311,77 @@ void CodeGenFunction::EmitOMPAggregateAssign(
EmitBlock(DoneBB, /*IsFinished=*/true);
}
+/// Check if the combiner is a call to UDR combiner and if it is so return the
+/// UDR decl used for reduction.
+static const OMPDeclareReductionDecl *
+getReductionInit(const Expr *ReductionOp) {
+ if (auto *CE = dyn_cast<CallExpr>(ReductionOp))
+ if (auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
+ if (auto *DRE =
+ dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
+ if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
+ return DRD;
+ return nullptr;
+}
+
+static void emitInitWithReductionInitializer(CodeGenFunction &CGF,
+ const OMPDeclareReductionDecl *DRD,
+ const Expr *InitOp,
+ Address Private, Address Original,
+ QualType Ty) {
+ if (DRD->getInitializer()) {
+ std::pair<llvm::Function *, llvm::Function *> Reduction =
+ CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD);
+ auto *CE = cast<CallExpr>(InitOp);
+ auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
+ const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts();
+ const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts();
+ auto *LHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
+ auto *RHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
+ CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
+ PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
+ [=]() -> Address { return Private; });
+ PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
+ [=]() -> Address { return Original; });
+ (void)PrivateScope.Privatize();
+ RValue Func = RValue::get(Reduction.second);
+ CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func);
+ CGF.EmitIgnoredExpr(InitOp);
+ } else {
+ llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty);
+ auto *GV = new llvm::GlobalVariable(
+ CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage, Init, ".init");
+ LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty);
+ RValue InitRVal;
+ switch (CGF.getEvaluationKind(Ty)) {
+ case TEK_Scalar:
+ InitRVal = CGF.EmitLoadOfLValue(LV, SourceLocation());
+ break;
+ case TEK_Complex:
+ InitRVal =
+ RValue::getComplex(CGF.EmitLoadOfComplex(LV, SourceLocation()));
+ break;
+ case TEK_Aggregate:
+ InitRVal = RValue::getAggregate(LV.getAddress());
+ break;
+ }
+ OpaqueValueExpr OVE(SourceLocation(), Ty, VK_RValue);
+ CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);
+ CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(),
+ /*IsInitializer=*/false);
+ }
+}
+
/// \brief Emit initialization of arrays of complex types.
/// \param DestAddr Address of the array.
/// \param Type Type of array.
/// \param Init Initial expression of array.
+/// \param SrcAddr Address of the original array.
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
- QualType Type, const Expr *Init) {
+ QualType Type, const Expr *Init,
+ Address SrcAddr = Address::invalid()) {
+ auto *DRD = getReductionInit(Init);
// Perform element-by-element initialization.
QualType ElementTy;
@@ -324,7 +390,13 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
auto NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr);
DestAddr =
CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType());
+ if (DRD)
+ SrcAddr =
+ CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
+ llvm::Value *SrcBegin = nullptr;
+ if (DRD)
+ SrcBegin = SrcAddr.getPointer();
auto DestBegin = DestAddr.getPointer();
// Cast from pointer to array type to pointer to single element.
auto DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements);
@@ -341,6 +413,16 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy);
+ llvm::PHINode *SrcElementPHI = nullptr;
+ Address SrcElementCurrent = Address::invalid();
+ if (DRD) {
+ SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2,
+ "omp.arraycpy.srcElementPast");
+ SrcElementPHI->addIncoming(SrcBegin, EntryBB);
+ SrcElementCurrent =
+ Address(SrcElementPHI,
+ SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
+ }
llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI(
DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
DestElementPHI->addIncoming(DestBegin, EntryBB);
@@ -351,8 +433,19 @@ static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr,
// Emit copy.
{
CodeGenFunction::RunCleanupsScope InitScope(CGF);
- CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
- /*IsInitializer=*/false);
+ if (DRD) {
+ emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent,
+ SrcElementCurrent, ElementTy);
+ } else
+ CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(),
+ /*IsInitializer=*/false);
+ }
+
+ if (DRD) {
+ // Shift the address forward by one element.
+ auto SrcElementNext = CGF.Builder.CreateConstGEP1_32(
+ SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
+ SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock());
}
// Shift the address forward by one element.
@@ -752,10 +845,12 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
auto ILHS = C->lhs_exprs().begin();
auto IRHS = C->rhs_exprs().begin();
auto IPriv = C->privates().begin();
+ auto IRed = C->reduction_ops().begin();
for (auto IRef : C->varlists()) {
auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
+ auto *DRD = getReductionInit(*IRed);
if (auto *OASE = dyn_cast<OMPArraySectionExpr>(IRef)) {
auto *Base = OASE->getBase()->IgnoreParenImpCasts();
while (auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base))
@@ -779,7 +874,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
// Emit reduction copy.
bool IsRegistered = PrivateScope.addPrivate(
OrigVD, [this, OrigVD, PrivateVD, BaseLValue, OASELValueLB,
- OASELValueUB, OriginalBaseLValue]() -> Address {
+ OASELValueUB, OriginalBaseLValue, DRD, IRed]() -> Address {
// Emit VarDecl with copy init for arrays.
// Get the address of the original variable captured in current
// captured region.
@@ -797,7 +892,9 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
auto Emission = EmitAutoVarAlloca(*PrivateVD);
auto Addr = Emission.getAllocatedAddress();
auto *Init = PrivateVD->getInit();
- EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), Init);
+ EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(),
+ DRD ? *IRed : Init,
+ OASELValueLB.getAddress());
EmitAutoVarCleanups(Emission);
// Emit private VarDecl with reduction init.
auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(),
@@ -831,10 +928,17 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
// Emit reduction copy.
bool IsRegistered = PrivateScope.addPrivate(
OrigVD, [this, OrigVD, PrivateVD, BaseLValue, ASELValue,
- OriginalBaseLValue]() -> Address {
+ OriginalBaseLValue, DRD, IRed]() -> Address {
// Emit private VarDecl with reduction init.
- EmitDecl(*PrivateVD);
- auto Addr = GetAddrOfLocalVar(PrivateVD);
+ AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
+ auto Addr = Emission.getAllocatedAddress();
+ if (DRD) {
+ emitInitWithReductionInitializer(*this, DRD, *IRed, Addr,
+ ASELValue.getAddress(),
+ ASELValue.getType());
+ } else
+ EmitAutoVarInit(Emission);
+ EmitAutoVarCleanups(Emission);
auto *Offset = Builder.CreatePtrDiff(BaseLValue.getPointer(),
ASELValue.getPointer());
auto *Ptr = Builder.CreateGEP(Addr.getPointer(), Offset);
@@ -859,11 +963,11 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
CapturedStmtInfo->lookup(OrigVD) != nullptr,
IRef->getType(), VK_LValue, IRef->getExprLoc());
Address OriginalAddr = EmitLValue(&DRE).getAddress();
- PrivateScope.addPrivate(LHSVD, [this, OriginalAddr,
+ PrivateScope.addPrivate(LHSVD, [this, &OriginalAddr,
LHSVD]() -> Address {
- return Builder.CreateElementBitCast(
- OriginalAddr, ConvertTypeForMem(LHSVD->getType()),
- "lhs.begin");
+ OriginalAddr = Builder.CreateElementBitCast(
+ OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
+ return OriginalAddr;
});
bool IsRegistered = PrivateScope.addPrivate(OrigVD, [&]() -> Address {
if (Type->isVariablyModifiedType()) {
@@ -879,7 +983,8 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
auto Emission = EmitAutoVarAlloca(*PrivateVD);
auto Addr = Emission.getAllocatedAddress();
auto *Init = PrivateVD->getInit();
- EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(), Init);
+ EmitOMPAggregateInit(*this, Addr, PrivateVD->getType(),
+ DRD ? *IRed : Init, OriginalAddr);
EmitAutoVarCleanups(Emission);
return Emission.getAllocatedAddress();
});
@@ -894,18 +999,29 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
} else {
// Store the address of the original variable associated with the LHS
// implicit variable.
- PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef]() -> Address {
+ Address OriginalAddr = Address::invalid();
+ PrivateScope.addPrivate(LHSVD, [this, OrigVD, IRef,
+ &OriginalAddr]() -> Address {
DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
CapturedStmtInfo->lookup(OrigVD) != nullptr,
IRef->getType(), VK_LValue, IRef->getExprLoc());
- return EmitLValue(&DRE).getAddress();
+ OriginalAddr = EmitLValue(&DRE).getAddress();
+ return OriginalAddr;
});
// Emit reduction copy.
- bool IsRegistered =
- PrivateScope.addPrivate(OrigVD, [this, PrivateVD]() -> Address {
+ bool IsRegistered = PrivateScope.addPrivate(
+ OrigVD, [this, PrivateVD, OriginalAddr, DRD, IRed]() -> Address {
// Emit private VarDecl with reduction init.
- EmitDecl(*PrivateVD);
- return GetAddrOfLocalVar(PrivateVD);
+ AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
+ auto Addr = Emission.getAllocatedAddress();
+ if (DRD) {
+ emitInitWithReductionInitializer(*this, DRD, *IRed, Addr,
+ OriginalAddr,
+ PrivateVD->getType());
+ } else
+ EmitAutoVarInit(Emission);
+ EmitAutoVarCleanups(Emission);
+ return Addr;
});
assert(IsRegistered && "private var already registered as private");
// Silence the warning about unused variable.
@@ -918,6 +1034,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
++ILHS;
++IRHS;
++IPriv;
+ ++IRed;
}
}
}
OpenPOWER on IntegriCloud