[OPENMP] Codegen for 'firstprivate' clause.

This patch generates some helper variables that used as private copies of the corresponding original variables inside an OpenMP 'parallel' directive. These generated variables are initialized by copy using values of the original variables (with the copy constructor, if any). For arrays, initializator is generated for single element and in the codegen procedure this initial value is automatically propagated between all elements of the private copy.
In outlined function, references to original variables are replaced by the references to these private helper variables. At the end of the initialization of the private variables an implicit barier is generated by calling __kmpc_barrier(...) runtime function to be sure that all threads were initialized using original values of the variables.
Differential Revision: http://reviews.llvm.org/D5140

llvm-svn: 219295

5 files changed, 270 insertions, 68 deletions

diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index d4a546dc112..24575ce6d41 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1073,7 +1073,7 @@ static bool isCapturedBy(const VarDecl &var, const Expr *e) {
 
 /// \brief Determine whether the given initializer is trivial in the sense
 /// that it requires no code to be generated.
-static bool isTrivialInitializer(const Expr *Init) {
+bool CodeGenFunction::isTrivialInitializer(const Expr *Init) {
   if (!Init)
     return true;
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 9dfcd0753b2..c439833cfe7 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -24,6 +24,29 @@
 using namespace clang;
 using namespace CodeGen;
 
+void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, Stmt *S) {
+  CodeGenFunction::OuterDeclMapTy OuterDeclMap;
+  CGF.EmitOMPFirstprivateClause(Directive, OuterDeclMap);
+  if (!OuterDeclMap.empty()) {
+    // Emit implicit barrier to synchronize threads and avoid data races.
+    auto Flags = static_cast<CGOpenMPRuntime::OpenMPLocationFlags>(
+        CGOpenMPRuntime::OMP_IDENT_KMPC |
+        CGOpenMPRuntime::OMP_IDENT_BARRIER_IMPL);
+    CGF.CGM.getOpenMPRuntime().EmitOMPBarrierCall(CGF, Directive.getLocStart(),
+                                                  Flags);
+    // Remap captured variables to use their private copies in the outlined
+    // function.
+    for (auto I : OuterDeclMap) {
+      CGF.LocalDeclMap[I.first] = I.second;
+    }
+  }
+  CGCapturedStmtInfo::EmitBody(CGF, S);
+  // Clear mappings of captured private variables.
+  for (auto I : OuterDeclMap) {
+    CGF.LocalDeclMap.erase(I.first);
+  }
+}
+
 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
     : CGM(CGM), DefaultOpenMPPSource(nullptr) {
   IdentTy = llvm::StructType::create(
@@ -51,11 +74,10 @@ CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) {
       DefaultOpenMPPSource =
           llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
     }
-    llvm::GlobalVariable *DefaultOpenMPLocation = cast<llvm::GlobalVariable>(
-        CGM.CreateRuntimeVariable(IdentTy, ".kmpc_default_loc.addr"));
+    auto DefaultOpenMPLocation = new llvm::GlobalVariable(
+        CGM.getModule(), IdentTy, /*isConstant*/ true,
+        llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr);
     DefaultOpenMPLocation->setUnnamedAddr(true);
-    DefaultOpenMPLocation->setConstant(true);
-    DefaultOpenMPLocation->setLinkage(llvm::GlobalValue::PrivateLinkage);
 
     llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true);
     llvm::Constant *Values[] = {Zero,
@@ -63,6 +85,7 @@ CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) {
                                 Zero, Zero, DefaultOpenMPPSource};
     llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values);
     DefaultOpenMPLocation->setInitializer(Init);
+    OpenMPDefaultLocMap[Flags] = DefaultOpenMPLocation;
     return DefaultOpenMPLocation;
   }
   return Entry;
@@ -120,14 +143,14 @@ llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation(
   return LocValue;
 }
 
-llvm::Value *CGOpenMPRuntime::GetOpenMPGlobalThreadNum(CodeGenFunction &CGF,
-                                                       SourceLocation Loc) {
+llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF,
+                                                SourceLocation Loc) {
   assert(CGF.CurFn && "No function in current CodeGenFunction.");
 
-  llvm::Value *GTid = nullptr;
-  OpenMPGtidMapTy::iterator I = OpenMPGtidMap.find(CGF.CurFn);
-  if (I != OpenMPGtidMap.end()) {
-    GTid = I->second;
+  llvm::Value *ThreadID = nullptr;
+  OpenMPThreadIDMapTy::iterator I = OpenMPThreadIDMap.find(CGF.CurFn);
+  if (I != OpenMPThreadIDMap.end()) {
+    ThreadID = I->second;
   } else {
     // Check if current function is a function which has first parameter
     // with type int32 and name ".global_tid.".
@@ -145,24 +168,24 @@ llvm::Value *CGOpenMPRuntime::GetOpenMPGlobalThreadNum(CodeGenFunction &CGF,
         CGF.CurFn->arg_begin()->getName() == ".global_tid.") {
       CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
       CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
-      GTid = CGF.Builder.CreateLoad(CGF.CurFn->arg_begin());
+      ThreadID = CGF.Builder.CreateLoad(CGF.CurFn->arg_begin());
     } else {
       // Generate "int32 .kmpc_global_thread_num.addr;"
       CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
       CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
       llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc)};
-      GTid = CGF.EmitRuntimeCall(
+      ThreadID = CGF.EmitRuntimeCall(
           CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), Args);
     }
-    OpenMPGtidMap[CGF.CurFn] = GTid;
+    OpenMPThreadIDMap[CGF.CurFn] = ThreadID;
   }
-  return GTid;
+  return ThreadID;
 }
 
 void CGOpenMPRuntime::FunctionFinished(CodeGenFunction &CGF) {
   assert(CGF.CurFn && "No function in current CodeGenFunction.");
-  if (OpenMPGtidMap.count(CGF.CurFn))
-    OpenMPGtidMap.erase(CGF.CurFn);
+  if (OpenMPThreadIDMap.count(CGF.CurFn))
+    OpenMPThreadIDMap.erase(CGF.CurFn);
   if (OpenMPLocMap.count(CGF.CurFn))
     OpenMPLocMap.erase(CGF.CurFn);
 }
@@ -219,10 +242,33 @@ CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) {
     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
     break;
   }
+  case OMPRTL__kmpc_barrier: {
+    // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
+    llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
+    llvm::FunctionType *FnTy =
+        llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
+    RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
+    break;
+  }
   }
   return RTLFn;
 }
 
+void CGOpenMPRuntime::EmitOMPParallelCall(CodeGenFunction &CGF,
+                                          SourceLocation Loc,
+                                          llvm::Value *OutlinedFn,
+                                          llvm::Value *CapturedStruct) {
+  // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/)
+  llvm::Value *Args[] = {
+      EmitOpenMPUpdateLocation(CGF, Loc),
+      CGF.Builder.getInt32(1), // Number of arguments after 'microtask' argument
+      // (there is only one additional argument - 'context')
+      CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()),
+      CGF.EmitCastToVoidPtr(CapturedStruct)};
+  auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_fork_call);
+  CGF.EmitRuntimeCall(RTLFn, Args);
+}
+
 llvm::Value *CGOpenMPRuntime::GetCriticalRegionLock(StringRef CriticalName) {
   SmallString<256> Buffer;
   llvm::raw_svector_ostream Out(Buffer);
@@ -245,7 +291,7 @@ void CGOpenMPRuntime::EmitOMPCriticalRegionStart(CodeGenFunction &CGF,
                                                  SourceLocation Loc) {
   // Prepare other arguments and build a call to __kmpc_critical
   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
-                         GetOpenMPGlobalThreadNum(CGF, Loc), RegionLock};
+                         GetOpenMPThreadID(CGF, Loc), RegionLock};
   auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_critical);
   CGF.EmitRuntimeCall(RTLFn, Args);
 }
@@ -255,8 +301,19 @@ void CGOpenMPRuntime::EmitOMPCriticalRegionEnd(CodeGenFunction &CGF,
                                                SourceLocation Loc) {
   // Prepare other arguments and build a call to __kmpc_end_critical
   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
-                         GetOpenMPGlobalThreadNum(CGF, Loc), RegionLock};
+                         GetOpenMPThreadID(CGF, Loc), RegionLock};
   auto RTLFn =
       CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_end_critical);
   CGF.EmitRuntimeCall(RTLFn, Args);
 }
+
+void CGOpenMPRuntime::EmitOMPBarrierCall(CodeGenFunction &CGF,
+                                         SourceLocation Loc,
+                                         OpenMPLocationFlags Flags) {
+  // Build call __kmpc_barrier(loc, thread_id)
+  llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, Flags),
+                         GetOpenMPThreadID(CGF, Loc)};
+  auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_barrier);
+  CGF.EmitRuntimeCall(RTLFn, Args);
+}
+
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index bba2c8c4d78..ec6115fb094 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -14,33 +14,49 @@
 #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
 #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIME_H
 
+#include "CodeGenFunction.h"
+#include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/Type.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 
-namespace llvm {
-class AllocaInst;
-class CallInst;
-class GlobalVariable;
-class Constant;
-class Function;
-class Module;
-class StructLayout;
-class ArrayType;
-class FunctionType;
-class StructType;
-class Type;
-class Value;
-} // namespace llvm
-
 namespace clang {
 
 namespace CodeGen {
 
-class CodeGenFunction;
-class CodeGenModule;
+/// \brief API for captured statement code generation in OpenMP constructs.
+class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
+public:
+  CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &S,
+                     const VarDecl *ThreadIDVar)
+      : CGCapturedStmtInfo(S, CR_OpenMP), ThreadIDVar(ThreadIDVar),
+        Directive(D) {}
+
+  virtual ~CGOpenMPRegionInfo() override{};
+
+  /// \brief Gets a variable or parameter for storing global thread id
+  /// inside OpenMP construct.
+  const VarDecl *getThreadIDVariable() const { return ThreadIDVar; }
+
+  static bool classof(const CGCapturedStmtInfo *Info) {
+    return Info->getKind() == CR_OpenMP;
+  }
+
+  /// \brief Emit the captured statement body.
+  virtual void EmitBody(CodeGenFunction &CGF, Stmt *S) override;
+
+  /// \brief Get the name of the capture helper.
+  virtual StringRef getHelperName() const override { return ".omp_outlined."; }
+
+private:
+  /// \brief A variable or parameter storing global thread id for OpenMP
+  /// constructs.
+  const VarDecl *ThreadIDVar;
+  /// \brief OpenMP executable directive associated with the region.
+  const OMPExecutableDirective &Directive;
+};
 
 class CGOpenMPRuntime {
 public:
@@ -76,7 +92,9 @@ public:
     OMPRTL__kmpc_critical,
     // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
     // kmp_critical_name *crit);
-    OMPRTL__kmpc_end_critical
+    OMPRTL__kmpc_end_critical,
+    // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
+    OMPRTL__kmpc_barrier
   };
 
 private:
@@ -139,24 +157,15 @@ private:
   /// \brief Map of local debug location and functions.
   typedef llvm::DenseMap<llvm::Function *, llvm::Value *> OpenMPLocMapTy;
   OpenMPLocMapTy OpenMPLocMap;
-  /// \brief Map of local gtid and functions.
-  typedef llvm::DenseMap<llvm::Function *, llvm::Value *> OpenMPGtidMapTy;
-  OpenMPGtidMapTy OpenMPGtidMap;
+  /// \brief Map of local ThreadID and functions.
+  typedef llvm::DenseMap<llvm::Function *, llvm::Value *> OpenMPThreadIDMapTy;
+  OpenMPThreadIDMapTy OpenMPThreadIDMap;
   /// \brief Type kmp_critical_name, originally defined as typedef kmp_int32
   /// kmp_critical_name[8];
   llvm::ArrayType *KmpCriticalNameTy;
   /// \brief Map of critical regions names and the corresponding lock objects.
   llvm::StringMap<llvm::Value *, llvm::BumpPtrAllocator> CriticalRegionVarNames;
 
-public:
-  explicit CGOpenMPRuntime(CodeGenModule &CGM);
-  virtual ~CGOpenMPRuntime() {}
-
-  /// \brief Cleans up references to the objects in finished function.
-  /// \param CGF Reference to finished CodeGenFunction.
-  ///
-  void FunctionFinished(CodeGenFunction &CGF);
-
   /// \brief Emits object of ident_t type with info for source location.
   /// \param CGF Reference to current CodeGenFunction.
   /// \param Loc Clang source location.
@@ -166,13 +175,6 @@ public:
   EmitOpenMPUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc,
                            OpenMPLocationFlags Flags = OMP_IDENT_KMPC);
 
-  /// \brief Generates global thread number value.
-  /// \param CGF Reference to current CodeGenFunction.
-  /// \param Loc Clang source location.
-  ///
-  llvm::Value *GetOpenMPGlobalThreadNum(CodeGenFunction &CGF,
-                                        SourceLocation Loc);
-
   /// \brief Returns pointer to ident_t type;
   llvm::Type *getIdentTyPointerTy();
 
@@ -184,6 +186,33 @@ public:
   /// \return Specified function.
   llvm::Constant *CreateRuntimeFunction(OpenMPRTLFunction Function);
 
+  /// \brief Gets thread id value for the current thread.
+  /// \param CGF Reference to current CodeGenFunction.
+  /// \param Loc Clang source location.
+  ///
+  llvm::Value *GetOpenMPThreadID(CodeGenFunction &CGF, SourceLocation Loc);
+
+public:
+  explicit CGOpenMPRuntime(CodeGenModule &CGM);
+  virtual ~CGOpenMPRuntime() {}
+
+  /// \brief Cleans up references to the objects in finished function.
+  /// \param CGF Reference to finished CodeGenFunction.
+  ///
+  void FunctionFinished(CodeGenFunction &CGF);
+
+  /// \brief Emits code for parallel call of the \a OutlinedFn with variables
+  /// captured in a record which address is stored in \a CapturedStruct.
+  /// \param CGF Reference to current CodeGenFunction.
+  /// \param Loc Clang source location.
+  /// \param OutlinedFn Outlined function to be run in parallel threads.
+  /// \param CapturedStruct A pointer to the record with the references to
+  /// variables used in \a OutlinedFn function.
+  ///
+  virtual void EmitOMPParallelCall(CodeGenFunction &CGF, SourceLocation Loc,
+                                   llvm::Value *OutlinedFn,
+                                   llvm::Value *CapturedStruct);
+
   /// \brief Returns corresponding lock object for the specified critical region
   /// name. If the lock object does not exist it is created, otherwise the
   /// reference to the existing copy is returned.
@@ -208,6 +237,14 @@ public:
   virtual void EmitOMPCriticalRegionEnd(CodeGenFunction &CGF,
                                         llvm::Value *RegionLock,
                                         SourceLocation Loc);
+
+  /// \brief Emits a barrier for OpenMP threads.
+  /// \param CGF Reference to current CodeGenFunction.
+  /// \param Loc Clang source location.
+  /// \param Flags Flags for the barrier.
+  ///
+  virtual void EmitOMPBarrierCall(CodeGenFunction &CGF, SourceLocation Loc,
+                                  OpenMPLocationFlags Flags);
 };
 } // namespace CodeGen
 } // namespace clang
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 985cc0e62ae..5efff2ccbde 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -24,6 +24,110 @@ using namespace CodeGen;
 //                              OpenMP Directive Emission
 //===----------------------------------------------------------------------===//
 
+void CodeGenFunction::EmitOMPAggregateAssign(LValue OriginalAddr,
+                                             llvm::Value *PrivateAddr,
+                                             const Expr *AssignExpr,
+                                             QualType OriginalType,
+                                             const VarDecl *VDInit) {
+  EmitBlock(createBasicBlock(".omp.assign.begin."));
+  if (!isa<CXXConstructExpr>(AssignExpr) || isTrivialInitializer(AssignExpr)) {
+    // Perform simple memcpy.
+    EmitAggregateAssign(PrivateAddr, OriginalAddr.getAddress(),
+                        AssignExpr->getType());
+  } else {
+    // Perform element-by-element initialization.
+    QualType ElementTy;
+    auto SrcBegin = OriginalAddr.getAddress();
+    auto DestBegin = PrivateAddr;
+    auto ArrayTy = OriginalType->getAsArrayTypeUnsafe();
+    auto SrcNumElements = emitArrayLength(ArrayTy, ElementTy, SrcBegin);
+    auto DestNumElements = emitArrayLength(ArrayTy, ElementTy, DestBegin);
+    auto SrcEnd = Builder.CreateGEP(SrcBegin, SrcNumElements);
+    auto DestEnd = Builder.CreateGEP(DestBegin, DestNumElements);
+    // The basic structure here is a do-while loop, because we don't
+    // need to check for the zero-element case.
+    auto BodyBB = createBasicBlock("omp.arraycpy.body");
+    auto DoneBB = createBasicBlock("omp.arraycpy.done");
+    auto IsEmpty =
+        Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
+    Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
+
+    // Enter the loop body, making that address the current address.
+    auto EntryBB = Builder.GetInsertBlock();
+    EmitBlock(BodyBB);
+    auto SrcElementPast = Builder.CreatePHI(SrcBegin->getType(), 2,
+                                            "omp.arraycpy.srcElementPast");
+    SrcElementPast->addIncoming(SrcEnd, EntryBB);
+    auto DestElementPast = Builder.CreatePHI(DestBegin->getType(), 2,
+                                             "omp.arraycpy.destElementPast");
+    DestElementPast->addIncoming(DestEnd, EntryBB);
+
+    // Shift the address back by one element.
+    auto NegativeOne = llvm::ConstantInt::get(SizeTy, -1, true);
+    auto DestElement = Builder.CreateGEP(DestElementPast, NegativeOne,
+                                         "omp.arraycpy.dest.element");
+    auto SrcElement = Builder.CreateGEP(SrcElementPast, NegativeOne,
+                                        "omp.arraycpy.src.element");
+    {
+      // Create RunCleanScope to cleanup possible temps.
+      CodeGenFunction::RunCleanupsScope Init(*this);
+      // Emit initialization for single element.
+      LocalDeclMap[VDInit] = SrcElement;
+      EmitAnyExprToMem(AssignExpr, DestElement,
+                       AssignExpr->getType().getQualifiers(),
+                       /*IsInitializer*/ false);
+      LocalDeclMap.erase(VDInit);
+    }
+
+    // Check whether we've reached the end.
+    auto Done =
+        Builder.CreateICmpEQ(DestElement, DestBegin, "omp.arraycpy.done");
+    Builder.CreateCondBr(Done, DoneBB, BodyBB);
+    DestElementPast->addIncoming(DestElement, Builder.GetInsertBlock());
+    SrcElementPast->addIncoming(SrcElement, Builder.GetInsertBlock());
+
+    // Done.
+    EmitBlock(DoneBB, true);
+  }
+  EmitBlock(createBasicBlock(".omp.assign.end."));
+}
+
+void CodeGenFunction::EmitOMPFirstprivateClause(
+    const OMPExecutableDirective &D,
+    CodeGenFunction::OuterDeclMapTy &OuterDeclMap) {
+  auto PrivateFilter = [](const OMPClause *C) -> bool {
+    return C->getClauseKind() == OMPC_firstprivate;
+  };
+  for (OMPExecutableDirective::filtered_clause_iterator<decltype(PrivateFilter)>
+           I(D.clauses(), PrivateFilter); I; ++I) {
+    auto *C = cast<OMPFirstprivateClause>(*I);
+    auto IRef = C->varlist_begin();
+    auto InitsRef = C->inits().begin();
+    for (auto IInit : C->private_copies()) {
+      auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
+      if (*InitsRef != nullptr) {
+        // Emit VarDecl with copy init for arrays.
+        auto *FD = CapturedStmtInfo->lookup(
+            cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl()));
+        LValue Base = MakeNaturalAlignAddrLValue(
+            CapturedStmtInfo->getContextValue(),
+            getContext().getTagDeclType(FD->getParent()));
+        auto OriginalAddr = EmitLValueForField(Base, FD);
+        auto VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
+        auto Emission = EmitAutoVarAlloca(*VD);
+        // Emit initialization of aggregate firstprivate vars.
+        EmitOMPAggregateAssign(OriginalAddr, Emission.getAllocatedAddress(),
+                               VD->getInit(), (*IRef)->getType(), VDInit);
+        EmitAutoVarCleanups(Emission);
+      } else
+        // Emit VarDecl with copy init.
+        EmitDecl(*VD);
+      OuterDeclMap[cast<DeclRefExpr>(*IRef)->getDecl()] = GetAddrOfLocalVar(VD);
+      ++IRef, ++InitsRef;
+    }
+  }
+}
+
 void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
   const CapturedStmt *CS = cast<CapturedStmt>(S.getAssociatedStmt());
   llvm::Value *CapturedStruct = GenerateCapturedStmtArgument(*CS);
@@ -31,22 +135,13 @@ void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
   llvm::Value *OutlinedFn;
   {
     CodeGenFunction CGF(CGM, true);
-    CGCapturedStmtInfo CGInfo(*CS, CS->getCapturedRegionKind());
+    CGOpenMPRegionInfo CGInfo(S, *CS, *CS->getCapturedDecl()->param_begin());
     CGF.CapturedStmtInfo = &CGInfo;
     OutlinedFn = CGF.GenerateCapturedStmtFunction(*CS);
   }
 
-  // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/)
-  llvm::Value *Args[] = {
-      CGM.getOpenMPRuntime().EmitOpenMPUpdateLocation(*this, S.getLocStart()),
-      Builder.getInt32(1), // Number of arguments after 'microtask' argument
-      // (there is only one additional argument - 'context')
-      Builder.CreateBitCast(OutlinedFn,
-                            CGM.getOpenMPRuntime().getKmpc_MicroPointerTy()),
-      EmitCastToVoidPtr(CapturedStruct)};
-  llvm::Constant *RTLFn = CGM.getOpenMPRuntime().CreateRuntimeFunction(
-      CGOpenMPRuntime::OMPRTL__kmpc_fork_call);
-  EmitRuntimeCall(RTLFn, Args);
+  CGM.getOpenMPRuntime().EmitOMPParallelCall(*this, S.getLocStart(), OutlinedFn,
+                                             CapturedStruct);
 }
 
 void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &S,
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index a4b231ddf4b..e471f157381 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -113,6 +113,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void operator=(const CodeGenFunction &) LLVM_DELETED_FUNCTION;
 
   friend class CGCXXABI;
+  friend class CGOpenMPRegionInfo;
 public:
   /// A jump destination is an abstract label, branching to which may
   /// require a jump out through normal cleanups.
@@ -1826,6 +1827,10 @@ public:
   typedef void SpecialInitFn(CodeGenFunction &Init, const VarDecl &D,
                              llvm::Value *Address);
 
+  /// \brief Determine whether the given initializer is trivial in the sense
+  /// that it requires no code to be generated.
+  bool isTrivialInitializer(const Expr *Init);
+
   /// EmitAutoVarDecl - Emit an auto variable declaration.
   ///
   /// This function can be called with a null (unreachable) insert point.
@@ -1991,8 +1996,16 @@ public:
                            ArrayRef<const Attr *> Attrs = None);
 
   llvm::Function *EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K);
+  void GenerateCapturedStmtFunctionProlog(const CapturedStmt &S);
+  llvm::Function *GenerateCapturedStmtFunctionEpilog(const CapturedStmt &S);
   llvm::Function *GenerateCapturedStmtFunction(const CapturedStmt &S);
   llvm::Value *GenerateCapturedStmtArgument(const CapturedStmt &S);
+  typedef llvm::DenseMap<const Decl *, llvm::Value *> OuterDeclMapTy;
+  void EmitOMPAggregateAssign(LValue OriginalAddr, llvm::Value *PrivateAddr,
+                              const Expr *AssignExpr, QualType Type,
+                              const VarDecl *VDInit);
+  void EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
+                                 OuterDeclMapTy &OuterDeclMap);
 
   void EmitOMPParallelDirective(const OMPParallelDirective &S);
   void EmitOMPSimdDirective(const OMPSimdDirective &S);