[OPENMP] Codegen for 'private' clause in 'task' directive.

For tasks codegen for private/firstprivate variables are different rather than for other directives.

1. Build an internal structure of privates for each private variable:
struct .kmp_privates_t. {
  Ty1 var1;
  ...
  Tyn varn;
};
2. Add a new field to kmp_task_t type with list of privates.
struct kmp_task_t {
  void *              shareds;
  kmp_routine_entry_t routine;
  kmp_int32           part_id;
  kmp_routine_entry_t destructors;
  .kmp_privates_t.    privates;
};
3. Create a function with destructors calls for all privates after end of task region.
kmp_int32 .omp_task_destructor.(kmp_int32 gtid, kmp_task_t *tt) {
  ~Destructor(&tt->privates.var1);
  ...
  ~Destructor(&tt->privates.varn);
  return 0;
}
4. Perform default initialization of all private fields (no initialization for POD data, default constructor calls for classes) + provide address of a destructor function after kmpc_omp_task_alloc() and before kmpc_omp_task() calls.
kmp_task_t *new_task = __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, kmp_routine_entry_t *task_entry);

DefaultConstructor(new_task->privates.var1);
new_task->shareds.var1_ref = &new_task->privates.var1;
...
DefaultConstructor(new_task->privates.varn);
new_task->shareds.varn_ref = &new_task->privates.varn;

new_task->destructors = .omp_task_destructor.;
kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t *new_task)


Differential Revision: http://reviews.llvm.org/D9322

llvm-svn: 236207

3 files changed, 183 insertions, 29 deletions

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index f0b0a2e34c5..1dbf0e55687 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1601,6 +1601,9 @@ enum KmpTaskTFields {
   KmpTaskTPartId,
   /// \brief Function with call of destructors for private variables.
   KmpTaskTDestructors,
+  /// \brief Record with list of all private/firstprivate copies for the task
+  /// directive.
+  KmpTaskTPrivates,
 };
 } // namespace
 
@@ -1626,9 +1629,35 @@ static void addFieldToRecordDecl(ASTContext &C, DeclContext *DC,
   DC->addDecl(Field);
 }
 
-static QualType createKmpTaskTRecordDecl(CodeGenModule &CGM,
-                                         QualType KmpInt32Ty,
-                                         QualType KmpRoutineEntryPointerQTy) {
+namespace {
+typedef std::pair<CharUnits /*Align*/,
+                  std::pair<const VarDecl *, const VarDecl *>> VDPair;
+} // namespace
+
+static RecordDecl *createPrivatesRecordDecl(CodeGenModule &CGM,
+                                            const ArrayRef<VDPair> Privates) {
+  if (!Privates.empty()) {
+    auto &C = CGM.getContext();
+    // Build struct .kmp_privates_t. {
+    //         /*  private vars  */
+    //       };
+    auto *RD = C.buildImplicitRecord(".kmp_privates.t");
+    RD->startDefinition();
+    for (auto &&Pair : Privates) {
+      addFieldToRecordDecl(C, RD,
+                           Pair.second.first->getType().getNonReferenceType());
+    }
+    // TODO: add firstprivate fields.
+    RD->completeDefinition();
+    return RD;
+  }
+  return nullptr;
+}
+
+static RecordDecl *createKmpTaskTRecordDecl(CodeGenModule &CGM,
+                                            QualType KmpInt32Ty,
+                                            QualType KmpRoutineEntryPointerQTy,
+                                            const ArrayRef<VDPair> Privates) {
   auto &C = CGM.getContext();
   // Build struct kmp_task_t {
   //         void *              shareds;
@@ -1643,9 +1672,11 @@ static QualType createKmpTaskTRecordDecl(CodeGenModule &CGM,
   addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
   addFieldToRecordDecl(C, RD, KmpInt32Ty);
   addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy);
-  // TODO: add private fields.
+  if (auto *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) {
+    addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD));
+  }
   RD->completeDefinition();
-  return C.getRecordType(RD);
+  return RD;
 }
 
 /// \brief Emit a proxy function which accepts kmp_task_t as the second
@@ -1695,13 +1726,12 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
       C.getTypeAlignInChars(KmpInt32Ty).getQuantity(), KmpInt32Ty, Loc);
   auto *SharedsPtr = CGF.Builder.CreateStructGEP(KmpTaskTTy, TaskTypeArgAddr,
                                                  /*Idx=*/KmpTaskTShareds);
-  auto *SharedsParam =
+  auto *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
       CGF.EmitLoadOfScalar(SharedsPtr, /*Volatile=*/false,
-                           CGM.PointerAlignInBytes, C.VoidPtrTy, Loc);
-  llvm::Value *CallArgs[] = {
-      GtidParam, PartidParam,
-      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-          SharedsParam, CGF.ConvertTypeForMem(SharedsPtrTy))};
+                           CGM.PointerAlignInBytes, C.VoidPtrTy, Loc),
+      CGF.ConvertTypeForMem(SharedsPtrTy));
+
+  llvm::Value *CallArgs[] = {GtidParam, PartidParam, SharedsParam};
   CGF.EmitCallOrInvoke(TaskFunction, CallArgs);
   CGF.EmitStoreThroughLValue(
       RValue::get(CGF.Builder.getInt32(/*C=*/0)),
@@ -1710,18 +1740,79 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
   return TaskEntry;
 }
 
+llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc,
+                                     QualType KmpInt32Ty,
+                                     QualType KmpTaskTPtrQTy,
+                                     QualType KmpTaskQTy,
+                                     RecordDecl *KmpTaskQTyRD) {
+  auto &C = CGM.getContext();
+  FunctionArgList Args;
+  ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty);
+  ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc,
+                                /*Id=*/nullptr, KmpTaskTPtrQTy);
+  Args.push_back(&GtidArg);
+  Args.push_back(&TaskTypeArg);
+  FunctionType::ExtInfo Info;
+  auto &DestructorFnInfo =
+      CGM.getTypes().arrangeFreeFunctionDeclaration(KmpInt32Ty, Args, Info,
+                                                    /*isVariadic=*/false);
+  auto *DestructorFnTy = CGM.getTypes().GetFunctionType(DestructorFnInfo);
+  auto *DestructorFn =
+      llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,
+                             ".omp_task_destructor.", &CGM.getModule());
+  CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, DestructorFnInfo, DestructorFn);
+  CodeGenFunction CGF(CGM);
+  CGF.disableDebugInfo();
+  CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo,
+                    Args);
+
+  auto *TaskTypeArgAddr = CGF.EmitLoadOfScalar(
+      CGF.GetAddrOfLocalVar(&TaskTypeArg), /*Volatile=*/false,
+      CGM.PointerAlignInBytes, KmpTaskTPtrQTy, Loc);
+  LValue Base = CGF.MakeNaturalAlignAddrLValue(TaskTypeArgAddr, KmpTaskQTy);
+  auto FI = std::next(KmpTaskQTyRD->field_begin(), KmpTaskTPrivates);
+  Base = CGF.EmitLValueForField(Base, *FI);
+  for (auto *Field :
+       cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
+    if (auto DtorKind = Field->getType().isDestructedType()) {
+      auto FieldLValue = CGF.EmitLValueForField(Base, Field);
+      CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
+    }
+  }
+  CGF.FinishFunction();
+  return DestructorFn;
+}
+
+static int array_pod_sort_comparator(const VDPair *P1, const VDPair *P2) {
+  return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0);
+}
+
 void CGOpenMPRuntime::emitTaskCall(
-    CodeGenFunction &CGF, SourceLocation Loc, bool Tied,
-    llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
+    CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+    bool Tied, llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
     llvm::Value *TaskFunction, QualType SharedsTy, llvm::Value *Shareds,
-    const Expr *IfCond) {
+    const Expr *IfCond, const ArrayRef<const Expr *> PrivateVars,
+    const ArrayRef<const Expr *> PrivateCopies) {
   auto &C = CGM.getContext();
+  llvm::SmallVector<VDPair, 8> Privates;
+  auto I = PrivateCopies.begin();
+  // Aggeregate privates and sort them by the alignment.
+  for (auto *E : PrivateVars) {
+    auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+    Privates.push_back(std::make_pair(
+        C.getTypeAlignInChars(VD->getType()),
+        std::make_pair(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()))));
+    ++I;
+  }
+  llvm::array_pod_sort(Privates.begin(), Privates.end(),
+                       array_pod_sort_comparator);
   auto KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
   // Build type kmp_routine_entry_t (if not built yet).
   emitKmpRoutineEntryT(KmpInt32Ty);
   // Build particular struct kmp_task_t for the given task.
-  auto KmpTaskQTy =
-      createKmpTaskTRecordDecl(CGM, KmpInt32Ty, KmpRoutineEntryPtrQTy);
+  auto *KmpTaskQTyRD = createKmpTaskTRecordDecl(
+      CGM, KmpInt32Ty, KmpRoutineEntryPtrQTy, Privates);
+  auto KmpTaskQTy = C.getRecordType(KmpTaskQTyRD);
   QualType KmpTaskTPtrQTy = C.getPointerType(KmpTaskQTy);
   auto *KmpTaskTTy = CGF.ConvertType(KmpTaskQTy);
   auto *KmpTaskTPtrTy = KmpTaskTTy->getPointerTo();
@@ -1762,18 +1853,57 @@ void CGOpenMPRuntime::emitTaskCall(
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(NewTask, KmpTaskTPtrTy);
   // Fill the data in the resulting kmp_task_t record.
   // Copy shareds if there are any.
+  auto *KmpTaskSharedsPtr = CGF.EmitLoadOfScalar(
+      CGF.Builder.CreateStructGEP(KmpTaskTTy, NewTaskNewTaskTTy,
+                                  /*Idx=*/KmpTaskTShareds),
+      /*Volatile=*/false, CGM.PointerAlignInBytes, SharedsPtrTy, Loc);
   if (!SharedsTy->getAsStructureType()->getDecl()->field_empty())
-    CGF.EmitAggregateCopy(
-        CGF.EmitLoadOfScalar(
-            CGF.Builder.CreateStructGEP(KmpTaskTTy, NewTaskNewTaskTTy,
-                                        /*Idx=*/KmpTaskTShareds),
-            /*Volatile=*/false, CGM.PointerAlignInBytes, SharedsPtrTy, Loc),
-        Shareds, SharedsTy);
-  // TODO: generate function with destructors for privates.
+    CGF.EmitAggregateCopy(KmpTaskSharedsPtr, Shareds, SharedsTy);
+  // Emit initial values for private copies (if any).
+  bool NeedsCleanup = false;
+  if (!Privates.empty()) {
+    LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, KmpTaskQTy);
+    auto FI = std::next(KmpTaskQTyRD->field_begin(), KmpTaskTPrivates);
+    Base = CGF.EmitLValueForField(Base, *FI);
+    FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
+    LValue SharedsBase = CGF.MakeNaturalAlignAddrLValue(
+        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+            KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)),
+        SharedsTy);
+    CodeGenFunction::CGCapturedStmtInfo CapturesInfo(
+        cast<CapturedStmt>(*D.getAssociatedStmt()));
+    for (auto &&Pair : Privates) {
+      auto *VD = Pair.second.second;
+      auto *Init = VD->getAnyInitializer();
+      LValue PrivateLValue = CGF.EmitLValueForField(Base, *FI);
+      if (Init) {
+        CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
+      }
+      NeedsCleanup = NeedsCleanup || FI->getType().isDestructedType();
+      // Copy addresses of privates to corresponding references in the list of
+      // captured variables.
+      //   ...
+      //   tt->shareds.var_addr = &tt->privates.private_var;
+      //   ...
+      auto *OriginalVD = Pair.second.first;
+      auto *SharedField = CapturesInfo.lookup(OriginalVD);
+      auto SharedRefLValue =
+          CGF.EmitLValueForFieldInitialization(SharedsBase, SharedField);
+      CGF.EmitStoreThroughLValue(RValue::get(PrivateLValue.getAddress()),
+                                 SharedRefLValue);
+      ++FI, ++I;
+    }
+  }
   // Provide pointer to function with destructors for privates.
+  llvm::Value *DestructorFn =
+      NeedsCleanup
+          ? emitDestructorsFunction(CGM, Loc, KmpInt32Ty, KmpTaskTPtrQTy,
+                                    KmpTaskQTy, KmpTaskQTyRD)
+          : llvm::ConstantPointerNull::get(
+                cast<llvm::PointerType>(KmpRoutineEntryPtrTy));
   CGF.Builder.CreateAlignedStore(
-      llvm::ConstantPointerNull::get(
-          cast<llvm::PointerType>(KmpRoutineEntryPtrTy)),
+      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(DestructorFn,
+                                                      KmpRoutineEntryPtrTy),
       CGF.Builder.CreateStructGEP(KmpTaskTTy, NewTaskNewTaskTTy,
                                   /*Idx=*/KmpTaskTDestructors),
       CGM.PointerAlignInBytes);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index b1c70705757..7c805626684 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -540,6 +540,7 @@ public:
   /// 4. Emit a call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid,
   /// kmp_task_t *new_task), where new_task is a resulting structure from
   /// previous items.
+  /// \param D Current task directive.
   /// \param Tied true if the task is tied (the task is tied to the thread that
   /// can suspend its task region), false - untied (the task is not tied to any
   /// thread).
@@ -553,10 +554,17 @@ public:
   /// TaskFunction.
   /// \param IfCond Not a nullptr if 'if' clause was specified, nullptr
   /// otherwise.
-  virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, bool Tied,
+  /// \param PrivateVars List of references to private variables for the task
+  /// directive.
+  /// \param PrivateCopies List of private copies for each private variable in
+  /// \a PrivateVars.
+  virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
+                            const OMPExecutableDirective &D, bool Tied,
                             llvm::PointerIntPair<llvm::Value *, 1, bool> Final,
                             llvm::Value *TaskFunction, QualType SharedsTy,
-                            llvm::Value *Shareds, const Expr *IfCond);
+                            llvm::Value *Shareds, const Expr *IfCond,
+                            const ArrayRef<const Expr *> PrivateVars,
+                            const ArrayRef<const Expr *> PrivateCopies);
 
   /// \brief Emit code for the directive that does not require outlining.
   ///
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 6a4b5ecaa36..1489685459a 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1373,9 +1373,25 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
   if (auto C = S.getSingleClause(OMPC_if)) {
     IfCond = cast<OMPIfClause>(C)->getCondition();
   }
-  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), Tied, Final,
+  // Get list of private variables.
+  llvm::SmallVector<const Expr *, 8> Privates;
+  llvm::SmallVector<const Expr *, 8> PrivateCopies;
+  llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
+  for (auto &&I = S.getClausesOfKind(OMPC_private); I; ++I) {
+    auto *C = cast<OMPPrivateClause>(*I);
+    auto IRef = C->varlist_begin();
+    for (auto *IInit : C->private_copies()) {
+      auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
+      if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
+        Privates.push_back(*IRef);
+        PrivateCopies.push_back(IInit);
+      }
+      ++IRef;
+    }
+  }
+  CGM.getOpenMPRuntime().emitTaskCall(*this, S.getLocStart(), S, Tied, Final,
                                       OutlinedFn, SharedsTy, CapturedStruct,
-                                      IfCond);
+                                      IfCond, Privates, PrivateCopies);
 }
 
 void CodeGenFunction::EmitOMPTaskyieldDirective(