summaryrefslogtreecommitdiffstats
path: root/clang/lib/CodeGen
diff options
context:
space:
mode:
Diffstat (limited to 'clang/lib/CodeGen')
-rw-r--r--clang/lib/CodeGen/CGOpenMPRuntime.cpp160
-rw-r--r--clang/lib/CodeGen/CGStmtOpenMP.cpp104
2 files changed, 202 insertions, 62 deletions
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 789302c8855..1238accf42d 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1719,7 +1719,8 @@ createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy,
/// argument.
/// \code
/// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) {
-/// TaskFunction(gtid, tt->part_id, tt->shareds);
+/// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map,
+/// tt->shareds);
/// return 0;
/// }
/// \endcode
@@ -1727,7 +1728,8 @@ static llvm::Value *
emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy,
QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy,
- QualType SharedsPtrTy, llvm::Value *TaskFunction) {
+ QualType SharedsPtrTy, llvm::Value *TaskFunction,
+ llvm::Value *TaskPrivatesMap) {
auto &C = CGM.getContext();
FunctionArgList Args;
ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
@@ -1748,18 +1750,19 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
CGF.disableDebugInfo();
CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args);
- // TaskFunction(gtid, tt->task_data.part_id, tt->task_data.shareds);
+ // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map,
+ // tt->task_data.shareds);
auto *GtidParam = CGF.EmitLoadOfScalar(
CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false,
C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc);
auto *TaskTypeArgAddr = CGF.Builder.CreateAlignedLoad(
CGF.GetAddrOfLocalVar(&TaskTypeArg), CGM.PointerAlignInBytes);
- LValue Base =
+ LValue TDBase =
CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskTWithPrivatesQTy);
auto *KmpTaskTWithPrivatesQTyRD =
cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl());
- Base =
- CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin());
+ LValue Base =
+ CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin());
auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl());
auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
auto PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI);
@@ -1771,7 +1774,18 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
CGF.EmitLoadOfLValue(SharedsLVal, Loc).getScalarVal(),
CGF.ConvertTypeForMem(SharedsPtrTy));
- llvm::Value *CallArgs[] = {GtidParam, PartidParam, SharedsParam};
+ auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
+ llvm::Value *PrivatesParam;
+ if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
+ auto PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI);
+ PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ PrivatesLVal.getAddress(), CGF.VoidPtrTy);
+ } else {
+ PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ }
+
+ llvm::Value *CallArgs[] = {GtidParam, PartidParam, PrivatesParam,
+ TaskPrivatesMap, SharedsParam};
CGF.EmitCallOrInvoke(TaskFunction, CallArgs);
CGF.EmitStoreThroughLValue(
RValue::get(CGF.Builder.getInt32(/*C=*/0)),
@@ -1825,6 +1839,90 @@ static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
return DestructorFn;
}
+/// \brief Emit a privates mapping function for correct handling of private and
+/// firstprivate variables.
+/// \code
+/// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1>
+/// **noalias priv1,..., <tyn> **noalias privn) {
+/// *priv1 = &.privates.priv1;
+/// ...;
+/// *privn = &.privates.privn;
+/// }
+/// \endcode
+static llvm::Value *
+emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
+ const ArrayRef<const Expr *> PrivateVars,
+ const ArrayRef<const Expr *> FirstprivateVars,
+ QualType PrivatesQTy,
+ const ArrayRef<PrivateDataTy> Privates) {
+ auto &C = CGM.getContext();
+ FunctionArgList Args;
+ ImplicitParamDecl TaskPrivatesArg(
+ C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
+ C.getPointerType(PrivatesQTy).withConst().withRestrict());
+ Args.push_back(&TaskPrivatesArg);
+ llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
+ unsigned Counter = 1;
+ for (auto *E: PrivateVars) {
+ Args.push_back(ImplicitParamDecl::Create(
+ C, /*DC=*/nullptr, Loc,
+ /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType()))
+ .withConst()
+ .withRestrict()));
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ PrivateVarsPos[VD] = Counter;
+ ++Counter;
+ }
+ for (auto *E : FirstprivateVars) {
+ Args.push_back(ImplicitParamDecl::Create(
+ C, /*DC=*/nullptr, Loc,
+ /*Id=*/nullptr, C.getPointerType(C.getPointerType(E->getType()))
+ .withConst()
+ .withRestrict()));
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ PrivateVarsPos[VD] = Counter;
+ ++Counter;
+ }
+ FunctionType::ExtInfo Info;
+ auto &TaskPrivatesMapFnInfo =
+ CGM.getTypes().arrangeFreeFunctionDeclaration(C.VoidTy, Args, Info,
+ /*isVariadic=*/false);
+ auto *TaskPrivatesMapTy =
+ CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo);
+ auto *TaskPrivatesMap = llvm::Function::Create(
+ TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage,
+ ".omp_task_privates_map.", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo,
+ TaskPrivatesMap);
+ TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
+ CodeGenFunction CGF(CGM);
+ CGF.disableDebugInfo();
+ CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
+ TaskPrivatesMapFnInfo, Args);
+
+ // *privi = &.privates.privi;
+ auto *TaskPrivatesArgAddr = CGF.Builder.CreateAlignedLoad(
+ CGF.GetAddrOfLocalVar(&TaskPrivatesArg), CGM.PointerAlignInBytes);
+ LValue Base =
+ CGF.MakeNaturalAlignAddrLValue(TaskPrivatesArgAddr, PrivatesQTy);
+ auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl());
+ Counter = 0;
+ for (auto *Field : PrivatesQTyRD->fields()) {
+ auto FieldLVal = CGF.EmitLValueForField(Base, Field);
+ auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
+ auto RefLVal = CGF.MakeNaturalAlignAddrLValue(CGF.GetAddrOfLocalVar(VD),
+ VD->getType());
+ auto RefLoadRVal = CGF.EmitLoadOfLValue(RefLVal, Loc);
+ CGF.EmitStoreOfScalar(
+ FieldLVal.getAddress(),
+ CGF.MakeNaturalAlignAddrLValue(RefLoadRVal.getScalarVal(),
+ RefLVal.getType()->getPointeeType()));
+ ++Counter;
+ }
+ CGF.FinishFunction();
+ return TaskPrivatesMap;
+}
+
static int array_pod_sort_comparator(const PrivateDataTy *P1,
const PrivateDataTy *P2) {
return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0);
@@ -1841,7 +1939,7 @@ void CGOpenMPRuntime::emitTaskCall(
const ArrayRef<const Expr *> FirstprivateInits) {
auto &C = CGM.getContext();
llvm::SmallVector<PrivateDataTy, 8> Privates;
- // Aggeregate privates and sort them by the alignment.
+ // Aggregate privates and sort them by the alignment.
auto I = PrivateCopies.begin();
for (auto *E : PrivateVars) {
auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
@@ -1885,11 +1983,27 @@ void CGOpenMPRuntime::emitTaskCall(
CGM.getSize(C.getTypeSizeInChars(KmpTaskTWithPrivatesQTy));
QualType SharedsPtrTy = C.getPointerType(SharedsTy);
+ // Emit initial values for private copies (if any).
+ llvm::Value *TaskPrivatesMap = nullptr;
+ auto *TaskPrivatesMapTy =
+ std::next(cast<llvm::Function>(TaskFunction)->getArgumentList().begin(),
+ 3)
+ ->getType();
+ if (!Privates.empty()) {
+ auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
+ TaskPrivatesMap = emitTaskPrivateMappingFunction(
+ CGM, Loc, PrivateVars, FirstprivateVars, FI->getType(), Privates);
+ TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ TaskPrivatesMap, TaskPrivatesMapTy);
+ } else {
+ TaskPrivatesMap = llvm::ConstantPointerNull::get(
+ cast<llvm::PointerType>(TaskPrivatesMapTy));
+ }
// Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid,
// kmp_task_t *tt);
auto *TaskEntry = emitProxyTaskFunction(
CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTy,
- KmpTaskTQTy, SharedsPtrTy, TaskFunction);
+ KmpTaskTQTy, SharedsPtrTy, TaskFunction, TaskPrivatesMap);
// Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid,
// kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds,
@@ -1937,10 +2051,13 @@ void CGOpenMPRuntime::emitTaskCall(
auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
auto PrivatesBase = CGF.EmitLValueForField(Base, *FI);
FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
- LValue SharedsBase = CGF.MakeNaturalAlignAddrLValue(
- CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
- SharedsTy);
+ LValue SharedsBase;
+ if (!FirstprivateVars.empty()) {
+ SharedsBase = CGF.MakeNaturalAlignAddrLValue(
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
+ SharedsTy);
+ }
CodeGenFunction::CGCapturedStmtInfo CapturesInfo(
cast<CapturedStmt>(*D.getAssociatedStmt()));
for (auto &&Pair : Privates) {
@@ -2000,22 +2117,7 @@ void CGOpenMPRuntime::emitTaskCall(
}
}
NeedsCleanup = NeedsCleanup || FI->getType().isDestructedType();
- // Copy addresses of privates to corresponding references in the list of
- // captured variables.
- // ...
- // tt->shareds.var_addr = &tt->privates.private_var;
- // ...
- auto *OriginalVD = Pair.second.Original;
- auto *SharedField = CapturesInfo.lookup(OriginalVD);
- auto SharedRefLValue =
- CGF.EmitLValueForFieldInitialization(SharedsBase, SharedField);
- CGF.EmitStoreThroughLValue(
- RValue::get(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- PrivateLValue.getAddress(), SharedRefLValue.getAddress()
- ->getType()
- ->getPointerElementType())),
- SharedRefLValue);
- ++FI, ++I;
+ ++FI;
}
}
// Provide pointer to function with destructors for privates.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 78412e47345..07fc6e966af 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1434,39 +1434,9 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
auto *PartId = std::next(I);
// The first function argument for tasks is a thread id, the second one is a
// part id (0 for tied tasks, >=0 for untied task).
- auto &&CodeGen = [PartId, &S](CodeGenFunction &CGF) {
- if (*PartId) {
- // TODO: emit code for untied tasks.
- }
- CGF.EmitStmt(cast<CapturedStmt>(S.getAssociatedStmt())->getCapturedStmt());
- };
- auto OutlinedFn =
- CGM.getOpenMPRuntime().emitTaskOutlinedFunction(S, *I, CodeGen);
- // Check if we should emit tied or untied task.
- bool Tied = !S.getSingleClause(OMPC_untied);
- // Check if the task is final
- llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
- if (auto *Clause = S.getSingleClause(OMPC_final)) {
- // If the condition constant folds and can be elided, try to avoid emitting
- // the condition and the dead arm of the if/else.
- auto *Cond = cast<OMPFinalClause>(Clause)->getCondition();
- bool CondConstant;
- if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
- Final.setInt(CondConstant);
- else
- Final.setPointer(EvaluateExprAsBool(Cond));
- } else {
- // By default the task is not final.
- Final.setInt(/*IntVal=*/false);
- }
- auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
- const Expr *IfCond = nullptr;
- if (auto C = S.getSingleClause(OMPC_if)) {
- IfCond = cast<OMPIfClause>(C)->getCondition();
- }
llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
// Get list of private variables.
- llvm::SmallVector<const Expr *, 8> Privates;
+ llvm::SmallVector<const Expr *, 8> PrivateVars;
llvm::SmallVector<const Expr *, 8> PrivateCopies;
for (auto &&I = S.getClausesOfKind(OMPC_private); I; ++I) {
auto *C = cast<OMPPrivateClause>(*I);
@@ -1474,7 +1444,7 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
for (auto *IInit : C->private_copies()) {
auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
- Privates.push_back(*IRef);
+ PrivateVars.push_back(*IRef);
PrivateCopies.push_back(IInit);
}
++IRef;
@@ -1499,9 +1469,77 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
++IRef, ++IElemInitRef;
}
}
+ auto &&CodeGen = [PartId, &S, &PrivateVars, &FirstprivateVars](
+ CodeGenFunction &CGF) {
+ // Set proper addresses for generated private copies.
+ auto *CS = cast<CapturedStmt>(S.getAssociatedStmt());
+ OMPPrivateScope Scope(CGF);
+ if (!PrivateVars.empty() || !FirstprivateVars.empty()) {
+ auto *CopyFn = CGF.Builder.CreateAlignedLoad(
+ CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)),
+ CGF.PointerAlignInBytes);
+ auto *PrivatesPtr = CGF.Builder.CreateAlignedLoad(
+ CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)),
+ CGF.PointerAlignInBytes);
+ // Map privates.
+ llvm::SmallVector<std::pair<const VarDecl *, llvm::Value *>, 16>
+ PrivatePtrs;
+ llvm::SmallVector<llvm::Value *, 16> CallArgs;
+ CallArgs.push_back(PrivatesPtr);
+ for (auto *E : PrivateVars) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ auto *PrivatePtr =
+ CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()));
+ PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
+ CallArgs.push_back(PrivatePtr);
+ }
+ for (auto *E : FirstprivateVars) {
+ auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ auto *PrivatePtr =
+ CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()));
+ PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
+ CallArgs.push_back(PrivatePtr);
+ }
+ CGF.EmitRuntimeCall(CopyFn, CallArgs);
+ for (auto &&Pair : PrivatePtrs) {
+ auto *Replacement =
+ CGF.Builder.CreateAlignedLoad(Pair.second, CGF.PointerAlignInBytes);
+ Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
+ }
+ }
+ (void)Scope.Privatize();
+ if (*PartId) {
+ // TODO: emit code for untied tasks.
+ }
+ CGF.EmitStmt(CS->getCapturedStmt());
+ };
+ auto OutlinedFn =
+ CGM.getOpenMPRuntime().emitTaskOutlinedFunction(S, *I, CodeGen);
+ // Check if we should emit tied or untied task.
+ bool Tied = !S.getSingleClause(OMPC_untied);
+ // Check if the task is final
+ llvm::PointerIntPair<llvm::Value *, 1, bool> Final;
+ if (auto *Clause = S.getSingleClause(OMPC_final)) {
+ // If the condition constant folds and can be elided, try to avoid emitting
+ // the condition and the dead arm of the if/else.
+ auto *Cond = cast<OMPFinalClause>(Clause)->getCondition();
+ bool CondConstant;
+ if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
+ Final.setInt(CondConstant);
+ else
+ Final.setPointer(EvaluateExprAsBool(Cond));
+ } else {
+ // By default the task is not final.
+ Final.setInt(/*IntVal=*/false);
+ }
+ auto SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
+ const Expr *IfCond = nullptr;
+ if (auto C = S.getSingleClause(OMPC_if)) {
+ IfCond = cast<OMPIfClause>(C)->getCondition();
+ }
CGM.getOpenMPRuntime().emitTaskCall(
*this, S.getLocStart(), S, Tied, Final, OutlinedFn, SharedsTy,
- CapturedStruct, IfCond, Privates, PrivateCopies, FirstprivateVars,
+ CapturedStruct, IfCond, PrivateVars, PrivateCopies, FirstprivateVars,
FirstprivateCopies, FirstprivateInits);
}
OpenPOWER on IntegriCloud