[CodeGen] Merge equivalent block copy/helper functions.

Clang generates copy and dispose helper functions for each block literal
on the stack. Often these functions are equivalent for different blocks.
This commit makes changes to merge equivalent copy and dispose helper
functions and reduce code size.

To enable merging equivalent copy/dispose functions, the captured object
infomation is encoded into the helper function name. This allows IRGen
to check whether an equivalent helper function has already been emitted
and reuse the function instead of generating a new helper function
whenever a block is defined. In addition, the helper functions are
marked as linkonce_odr to enable merging helper functions that have the
same name across translation units and marked as unnamed_addr to enable
the linker's deduplication pass to merge functions that have different
names but the same content.

rdar://problem/42640608

Differential Revision: https://reviews.llvm.org/D50152

llvm-svn: 339438

10 files changed, 289 insertions, 78 deletions

diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 5eb3e9ac020..7d9215802ac 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -2503,21 +2503,24 @@ const ObjCInterfaceDecl *ASTContext::getObjContainingInterface(
 
 /// Get the copy initialization expression of VarDecl, or nullptr if
 /// none exists.
-Expr *ASTContext::getBlockVarCopyInits(const VarDecl*VD) {
+ASTContext::BlockVarCopyInit
+ASTContext::getBlockVarCopyInit(const VarDecl*VD) const {
   assert(VD && "Passed null params");
   assert(VD->hasAttr<BlocksAttr>() &&
          "getBlockVarCopyInits - not __block var");
-  llvm::DenseMap<const VarDecl*, Expr*>::iterator
-    I = BlockVarCopyInits.find(VD);
-  return (I != BlockVarCopyInits.end()) ? I->second : nullptr;
+  auto I = BlockVarCopyInits.find(VD);
+  if (I != BlockVarCopyInits.end())
+    return I->second;
+  return {nullptr, false};
 }
 
 /// Set the copy inialization expression of a block var decl.
-void ASTContext::setBlockVarCopyInits(VarDecl*VD, Expr* Init) {
-  assert(VD && Init && "Passed null params");
+void ASTContext::setBlockVarCopyInit(const VarDecl*VD, Expr *CopyExpr,
+                                     bool CanThrow) {
+  assert(VD && CopyExpr && "Passed null params");
   assert(VD->hasAttr<BlocksAttr>() &&
          "setBlockVarCopyInits - not __block var");
-  BlockVarCopyInits[VD] = Init;
+  BlockVarCopyInits[VD].setExprAndFlag(CopyExpr, CanThrow);
 }
 
 TypeSourceInfo *ASTContext::CreateTypeSourceInfo(QualType T,
@@ -5962,7 +5965,7 @@ LangAS ASTContext::getOpenCLTypeAddrSpace(const Type *T) const {
 bool ASTContext::BlockRequiresCopying(QualType Ty,
                                       const VarDecl *D) {
   if (const CXXRecordDecl *record = Ty->getAsCXXRecordDecl()) {
-    const Expr *copyExpr = getBlockVarCopyInits(D);
+    const Expr *copyExpr = getBlockVarCopyInit(D).getCopyExpr();
     if (!copyExpr && record->hasTrivialDestructor()) return false;
 
     return true;
diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index 5d03477e9bd..b1dbb505b5d 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CGBlocks.h"
+#include "CGCXXABI.h"
 #include "CGDebugInfo.h"
 #include "CGObjCRuntime.h"
 #include "CGOpenCLRuntime.h"
@@ -25,6 +26,7 @@
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Module.h"
+#include "llvm/Support/ScopedPrinter.h"
 #include <algorithm>
 #include <cstdio>
 
@@ -34,8 +36,8 @@ using namespace CodeGen;
 CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
   : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
     HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false),
-    LocalAddress(Address::invalid()), StructureType(nullptr), Block(block),
-    DominatingIP(nullptr) {
+    CapturesNonExternalType(false), LocalAddress(Address::invalid()),
+    StructureType(nullptr), Block(block), DominatingIP(nullptr) {
 
   // Skip asm prefix, if any.  'name' is usually taken directly from
   // the mangled name of the enclosing function.
@@ -464,6 +466,8 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
     } else if (CI.hasCopyExpr()) {
       info.NeedsCopyDispose = true;
       info.HasCXXObject = true;
+      if (!variable->getType()->getAsCXXRecordDecl()->isExternallyVisible())
+        info.CapturesNonExternalType = true;
 
     // So do C structs that require non-trivial copy construction or
     // destruction.
@@ -480,6 +484,8 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
         if (!record->hasTrivialDestructor()) {
           info.HasCXXObject = true;
           info.NeedsCopyDispose = true;
+          if (!record->isExternallyVisible())
+            info.CapturesNonExternalType = true;
         }
       }
     }
@@ -1522,13 +1528,17 @@ enum class BlockCaptureEntityKind {
 struct BlockCaptureManagedEntity {
   BlockCaptureEntityKind Kind;
   BlockFieldFlags Flags;
-  const BlockDecl::Capture &CI;
-  const CGBlockInfo::Capture &Capture;
+  const BlockDecl::Capture *CI;
+  const CGBlockInfo::Capture *Capture;
 
   BlockCaptureManagedEntity(BlockCaptureEntityKind Type, BlockFieldFlags Flags,
                             const BlockDecl::Capture &CI,
                             const CGBlockInfo::Capture &Capture)
-      : Kind(Type), Flags(Flags), CI(CI), Capture(Capture) {}
+      : Kind(Type), Flags(Flags), CI(&CI), Capture(&Capture) {}
+
+  bool operator<(const BlockCaptureManagedEntity &Other) const {
+    return Capture->getOffset() < Other.Capture->getOffset();
+  }
 };
 
 } // end anonymous namespace
@@ -1607,6 +1617,9 @@ static void findBlockCapturedManagedEntities(
     if (Info.first != BlockCaptureEntityKind::None)
       ManagedCaptures.emplace_back(Info.first, Info.second, CI, Capture);
   }
+
+  // Sort the captures by offset.
+  llvm::sort(ManagedCaptures.begin(), ManagedCaptures.end());
 }
 
 namespace {
@@ -1614,10 +1627,12 @@ namespace {
 struct CallBlockRelease final : EHScopeStack::Cleanup {
   Address Addr;
   BlockFieldFlags FieldFlags;
-  bool LoadBlockVarAddr;
+  bool LoadBlockVarAddr, CanThrow;
 
-  CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue)
-      : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue) {}
+  CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,
+                   bool CT)
+      : Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),
+        CanThrow(CT) {}
 
   void Emit(CodeGenFunction &CGF, Flags flags) override {
     llvm::Value *BlockVarAddr;
@@ -1628,15 +1643,112 @@ struct CallBlockRelease final : EHScopeStack::Cleanup {
       BlockVarAddr = Addr.getPointer();
     }
 
-    CGF.BuildBlockRelease(BlockVarAddr, FieldFlags);
+    CGF.BuildBlockRelease(BlockVarAddr, FieldFlags, CanThrow);
   }
 };
 } // end anonymous namespace
 
+/// Check if \p T is a C++ class that has a destructor that can throw.
+bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {
+  if (const auto *RD = T->getAsCXXRecordDecl())
+    if (const CXXDestructorDecl *DD = RD->getDestructor())
+      return DD->getType()->getAs<FunctionProtoType>()->canThrow();
+  return false;
+}
+
+static std::string getCopyDestroyHelperFuncName(
+    const SmallVectorImpl<BlockCaptureManagedEntity> &Captures,
+    CharUnits BlockAlignment, bool IsCopyHelper, CodeGenModule &CGM) {
+  ASTContext &Ctx = CGM.getContext();
+  std::unique_ptr<ItaniumMangleContext> MC(
+      ItaniumMangleContext::create(Ctx, Ctx.getDiagnostics()));
+
+  std::string Name =
+      IsCopyHelper ? "__copy_helper_block_" : "__destroy_helper_block_";
+  if (CGM.getLangOpts().Exceptions)
+    Name += "e";
+  if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
+    Name += "a";
+  Name += llvm::to_string(BlockAlignment.getQuantity()) + "_";
+
+  for (const BlockCaptureManagedEntity &E : Captures) {
+    const BlockDecl::Capture &CI = *E.CI;
+    BlockFieldFlags Flags = E.Flags;
+    QualType CaptureTy = CI.getVariable()->getType();
+    Name += llvm::to_string(E.Capture->getOffset().getQuantity());
+
+    switch (E.Kind) {
+    case BlockCaptureEntityKind::CXXRecord: {
+      Name += "c";
+      SmallString<256> Str;
+      llvm::raw_svector_ostream Out(Str);
+      MC->mangleTypeName(CaptureTy, Out);
+      Name += llvm::to_string(Str.size()) + Str.c_str();
+      break;
+    }
+    case BlockCaptureEntityKind::ARCWeak:
+      Name += "w";
+      break;
+    case BlockCaptureEntityKind::ARCStrong:
+      Name += "s";
+      break;
+    case BlockCaptureEntityKind::BlockObject: {
+      const VarDecl *Var = CI.getVariable();
+      unsigned F = Flags.getBitMask();
+      if (F & BLOCK_FIELD_IS_BYREF) {
+        Name += "r";
+        if (F & BLOCK_FIELD_IS_WEAK)
+          Name += "w";
+        else {
+          if (IsCopyHelper) {
+            if (Ctx.getBlockVarCopyInit(Var).canThrow())
+              Name += "c";
+          } else {
+            if (CodeGenFunction::cxxDestructorCanThrow(CaptureTy))
+              Name += "d";
+          }
+        }
+      } else {
+        assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");
+        if (F == BLOCK_FIELD_IS_BLOCK)
+          Name += "b";
+        else
+          Name += "o";
+      }
+      break;
+    }
+    case BlockCaptureEntityKind::NonTrivialCStruct: {
+      bool IsVolatile = CaptureTy.isVolatileQualified();
+      CharUnits Alignment =
+          BlockAlignment.alignmentAtOffset(E.Capture->getOffset());
+
+      Name += "n";
+      std::string Str;
+      if (IsCopyHelper)
+        Str = CodeGenFunction::getNonTrivialCopyConstructorStr(
+            CaptureTy, Alignment, IsVolatile, Ctx);
+      else
+        Str = CodeGenFunction::getNonTrivialDestructorStr(CaptureTy, Alignment,
+                                                          IsVolatile, Ctx);
+      // The underscore is necessary here because non-trivial copy constructor
+      // and destructor strings can start with a number.
+      Name += llvm::to_string(Str.size()) + "_" + Str;
+      break;
+    }
+    case BlockCaptureEntityKind::None:
+      llvm_unreachable("unexpected block capture kind");
+    }
+  }
+
+  return Name;
+}
+
 static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
                                Address Field, QualType CaptureType,
-                               BlockFieldFlags Flags, bool EHOnly,
-                               CodeGenFunction &CGF) {
+                               BlockFieldFlags Flags, bool ForCopyHelper,
+                               VarDecl *Var, CodeGenFunction &CGF) {
+  bool EHOnly = ForCopyHelper;
+
   switch (CaptureKind) {
   case BlockCaptureEntityKind::CXXRecord:
   case BlockCaptureEntityKind::ARCWeak:
@@ -1658,7 +1770,13 @@ static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
   case BlockCaptureEntityKind::BlockObject: {
     if (!EHOnly || CGF.getLangOpts().Exceptions) {
       CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;
-      CGF.enterByrefCleanup(Kind, Field, Flags, /*LoadBlockVarAddr*/ true);
+      // Calls to _Block_object_dispose along the EH path in the copy helper
+      // function don't throw as newly-copied __block variables always have a
+      // reference count of 2.
+      bool CanThrow =
+          !ForCopyHelper && CGF.cxxDestructorCanThrow(CaptureType);
+      CGF.enterByrefCleanup(Kind, Field, Flags, /*LoadBlockVarAddr*/ true,
+                            CanThrow);
     }
     break;
   }
@@ -1667,6 +1785,19 @@ static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
   }
 }
 
+static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,
+                                               llvm::Function *Fn,
+                                               const CGFunctionInfo &FI,
+                                               CodeGenModule &CGM) {
+  if (CapturesNonExternalType) {
+    CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
+  } else {
+    Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
+    Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
+    CGM.SetLLVMFunctionAttributes(nullptr, FI, Fn);
+    CGM.SetLLVMFunctionAttributesForDefinition(nullptr, Fn);
+  }
+}
 /// Generate the copy-helper function for a block closure object:
 ///   static void block_copy_helper(block_t *dst, block_t *src);
 /// The runtime will have previously initialized 'dst' by doing a
@@ -1677,6 +1808,16 @@ static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
 /// the contents of an individual __block variable to the heap.
 llvm::Constant *
 CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
+  SmallVector<BlockCaptureManagedEntity, 4> CopiedCaptures;
+  findBlockCapturedManagedEntities(blockInfo, getLangOpts(), CopiedCaptures,
+                                   computeCopyInfoForBlockCapture);
+  std::string FuncName =
+      getCopyDestroyHelperFuncName(CopiedCaptures, blockInfo.BlockAlign,
+                                   /*IsCopyHelper*/ true, CGM);
+
+  if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName))
+    return Func;
+
   ASTContext &C = getContext();
 
   FunctionArgList args;
@@ -1695,11 +1836,11 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
   llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);
 
   llvm::Function *Fn =
-    llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
-                           "__copy_helper_block_", &CGM.getModule());
+    llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage,
+                           FuncName, &CGM.getModule());
 
   IdentifierInfo *II
-    = &CGM.getContext().Idents.get("__copy_helper_block_");
+    = &CGM.getContext().Idents.get(FuncName);
 
   FunctionDecl *FD = FunctionDecl::Create(C,
                                           C.getTranslationUnitDecl(),
@@ -1709,8 +1850,8 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
                                           false,
                                           false);
 
-  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
-
+  setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI,
+                                     CGM);
   StartFunction(FD, C.VoidTy, Fn, FI, args);
   ApplyDebugLocation NL{*this, blockInfo.getBlockExpr()->getBeginLoc()};
   llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
@@ -1723,13 +1864,9 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
   dst = Address(Builder.CreateLoad(dst), blockInfo.BlockAlign);
   dst = Builder.CreateBitCast(dst, structPtrTy, "block.dest");
 
-  SmallVector<BlockCaptureManagedEntity, 4> CopiedCaptures;
-  findBlockCapturedManagedEntities(blockInfo, getLangOpts(), CopiedCaptures,
-                                   computeCopyInfoForBlockCapture);
-
   for (const auto &CopiedCapture : CopiedCaptures) {
-    const BlockDecl::Capture &CI = CopiedCapture.CI;
-    const CGBlockInfo::Capture &capture = CopiedCapture.Capture;
+    const BlockDecl::Capture &CI = *CopiedCapture.CI;
+    const CGBlockInfo::Capture &capture = *CopiedCapture.Capture;
     QualType captureType = CI.getVariable()->getType();
     BlockFieldFlags flags = CopiedCapture.Flags;
 
@@ -1783,28 +1920,17 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
           dstAddr, srcValue, llvm::ConstantInt::get(Int32Ty, flags.getBitMask())
         };
 
-        const VarDecl *variable = CI.getVariable();
-        bool copyCanThrow = false;
-        if (CI.isByRef() && variable->getType()->getAsCXXRecordDecl()) {
-          const Expr *copyExpr =
-            CGM.getContext().getBlockVarCopyInits(variable);
-          if (copyExpr) {
-            copyCanThrow = true; // FIXME: reuse the noexcept logic
-          }
-        }
-
-        if (copyCanThrow) {
+        if (CI.isByRef() && C.getBlockVarCopyInit(CI.getVariable()).canThrow())
           EmitRuntimeCallOrInvoke(CGM.getBlockObjectAssign(), args);
-        } else {
+        else
           EmitNounwindRuntimeCall(CGM.getBlockObjectAssign(), args);
-        }
       }
     }
 
     // Ensure that we destroy the copied object if an exception is thrown later
     // in the helper function.
-    pushCaptureCleanup(CopiedCapture.Kind, dstField, captureType, flags, /*EHOnly*/ true,
-                       *this);
+    pushCaptureCleanup(CopiedCapture.Kind, dstField, captureType, flags,
+                       /*ForCopyHelper*/ true, CI.getVariable(), *this);
   }
 
   FinishFunction();
@@ -1868,6 +1994,16 @@ computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
 /// variable.
 llvm::Constant *
 CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
+  SmallVector<BlockCaptureManagedEntity, 4> DestroyedCaptures;
+  findBlockCapturedManagedEntities(blockInfo, getLangOpts(), DestroyedCaptures,
+                                   computeDestroyInfoForBlockCapture);
+  std::string FuncName =
+      getCopyDestroyHelperFuncName(DestroyedCaptures, blockInfo.BlockAlign,
+                                   /*IsCopyHelper*/ false, CGM);
+
+  if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(FuncName))
+    return Func;
+
   ASTContext &C = getContext();
 
   FunctionArgList args;
@@ -1883,11 +2019,11 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
   llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(FI);
 
   llvm::Function *Fn =
-    llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
-                           "__destroy_helper_block_", &CGM.getModule());
+    llvm::Function::Create(LTy, llvm::GlobalValue::LinkOnceODRLinkage,
+                           FuncName, &CGM.getModule());
 
   IdentifierInfo *II
-    = &CGM.getContext().Idents.get("__destroy_helper_block_");
+    = &CGM.getContext().Idents.get(FuncName);
 
   FunctionDecl *FD = FunctionDecl::Create(C, C.getTranslationUnitDecl(),
                                           SourceLocation(),
@@ -1895,9 +2031,11 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
                                           nullptr, SC_Static,
                                           false, false);
 
-  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
-
+  setBlockHelperAttributesVisibility(blockInfo.CapturesNonExternalType, Fn, FI,
+                                     CGM);
   StartFunction(FD, C.VoidTy, Fn, FI, args);
+  markAsIgnoreThreadCheckingAtRuntime(Fn);
+
   ApplyDebugLocation NL{*this, blockInfo.getBlockExpr()->getBeginLoc()};
 
   llvm::Type *structPtrTy = blockInfo.StructureType->getPointerTo();
@@ -1908,20 +2046,17 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
 
   CodeGenFunction::RunCleanupsScope cleanups(*this);
 
-  SmallVector<BlockCaptureManagedEntity, 4> DestroyedCaptures;
-  findBlockCapturedManagedEntities(blockInfo, getLangOpts(), DestroyedCaptures,
-                                   computeDestroyInfoForBlockCapture);
-
   for (const auto &DestroyedCapture : DestroyedCaptures) {
-    const BlockDecl::Capture &CI = DestroyedCapture.CI;
-    const CGBlockInfo::Capture &capture = DestroyedCapture.Capture;
+    const BlockDecl::Capture &CI = *DestroyedCapture.CI;
+    const CGBlockInfo::Capture &capture = *DestroyedCapture.Capture;
     BlockFieldFlags flags = DestroyedCapture.Flags;
 
     Address srcField =
       Builder.CreateStructGEP(src, capture.getIndex(), capture.getOffset());
 
     pushCaptureCleanup(DestroyedCapture.Kind, srcField,
-                       CI.getVariable()->getType(), flags, /*EHOnly*/ false, *this);
+                       CI.getVariable()->getType(), flags,
+                       /*ForCopyHelper*/ false, CI.getVariable(), *this);
   }
 
   cleanups.ForceCleanup();
@@ -1961,7 +2096,7 @@ public:
     field = CGF.Builder.CreateBitCast(field, CGF.Int8PtrTy->getPointerTo(0));
     llvm::Value *value = CGF.Builder.CreateLoad(field);
 
-    CGF.BuildBlockRelease(value, Flags | BLOCK_BYREF_CALLER);
+    CGF.BuildBlockRelease(value, Flags | BLOCK_BYREF_CALLER, false);
   }
 
   void profileImpl(llvm::FoldingSetNodeID &id) const override {
@@ -2288,7 +2423,8 @@ CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
     byrefInfo.ByrefAlignment.alignmentAtOffset(byrefInfo.FieldOffset);
 
   if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
-    const Expr *copyExpr = CGM.getContext().getBlockVarCopyInits(&var);
+    const Expr *copyExpr =
+        CGM.getContext().getBlockVarCopyInit(&var).getCopyExpr();
     if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
 
     return ::buildByrefHelpers(
@@ -2591,19 +2727,25 @@ void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
   }
 }
 
-void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags) {
+void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,
+                                        bool CanThrow) {
   llvm::Value *F = CGM.getBlockObjectDispose();
   llvm::Value *args[] = {
     Builder.CreateBitCast(V, Int8PtrTy),
     llvm::ConstantInt::get(Int32Ty, flags.getBitMask())
   };
-  EmitNounwindRuntimeCall(F, args); // FIXME: throwing destructors?
+
+  if (CanThrow)
+    EmitRuntimeCallOrInvoke(F, args);
+  else
+    EmitNounwindRuntimeCall(F, args);
 }
 
 void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,
                                         BlockFieldFlags Flags,
-                                        bool LoadBlockVarAddr) {
-  EHStack.pushCleanup<CallBlockRelease>(Kind, Addr, Flags, LoadBlockVarAddr);
+                                        bool LoadBlockVarAddr, bool CanThrow) {
+  EHStack.pushCleanup<CallBlockRelease>(Kind, Addr, Flags, LoadBlockVarAddr,
+                                        CanThrow);
 }
 
 /// Adjust the declaration of something from the blocks API.
diff --git a/clang/lib/CodeGen/CGBlocks.h b/clang/lib/CodeGen/CGBlocks.h
index 5abf82b3f6e..402acea70e7 100644
--- a/clang/lib/CodeGen/CGBlocks.h
+++ b/clang/lib/CodeGen/CGBlocks.h
@@ -231,6 +231,11 @@ public:
   /// and their layout meta-data has been generated.
   bool HasCapturedVariableLayout : 1;
 
+  /// Indicates whether an object of a non-external C++ class is captured. This
+  /// bit is used to determine the linkage of the block copy/destroy helper
+  /// functions.
+  bool CapturesNonExternalType : 1;
+
   /// The mapping of allocated indexes within the block.
   llvm::DenseMap<const VarDecl*, Capture> Captures;
 
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 3a5bc0e4694..5c47212f1b5 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -1726,7 +1726,8 @@ void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) {
     if (emission.Variable->getType().isObjCGCWeak())
       Flags |= BLOCK_FIELD_IS_WEAK;
     enterByrefCleanup(NormalAndEHCleanup, emission.Addr, Flags,
-                      /*LoadBlockVarAddr*/ false);
+                      /*LoadBlockVarAddr*/ false,
+                      cxxDestructorCanThrow(emission.Variable->getType()));
   }
 }
 
diff --git a/clang/lib/CodeGen/CGNonTrivialStruct.cpp b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
index 922e0934b86..e9f60a9113c 100644
--- a/clang/lib/CodeGen/CGNonTrivialStruct.cpp
+++ b/clang/lib/CodeGen/CGNonTrivialStruct.cpp
@@ -283,8 +283,9 @@ struct GenDefaultInitializeFuncName
 struct GenDestructorFuncName : GenUnaryFuncName<GenDestructorFuncName>,
                                DestructedTypeVisitor<GenDestructorFuncName> {
   using Super = DestructedTypeVisitor<GenDestructorFuncName>;
-  GenDestructorFuncName(CharUnits DstAlignment, ASTContext &Ctx)
-      : GenUnaryFuncName<GenDestructorFuncName>("__destructor_", DstAlignment,
+  GenDestructorFuncName(const char *Prefix, CharUnits DstAlignment,
+                        ASTContext &Ctx)
+      : GenUnaryFuncName<GenDestructorFuncName>(Prefix, DstAlignment,
                                                 Ctx) {}
   void visitWithKind(QualType::DestructionKind DK, QualType FT,
                      const FieldDecl *FD, CharUnits CurStructOffset) {
@@ -824,11 +825,28 @@ void CodeGenFunction::callCStructDefaultConstructor(LValue Dst) {
                       IsVolatile, *this, std::array<Address, 1>({{DstPtr}}));
 }
 
+std::string
+CodeGenFunction::getNonTrivialCopyConstructorStr(QualType QT,
+                                                 CharUnits Alignment,
+                                                 bool IsVolatile,
+                                                 ASTContext &Ctx) {
+  GenBinaryFuncName<false> GenName("", Alignment, Alignment, Ctx);
+  return GenName.getName(QT, IsVolatile);
+}
+
+std::string
+CodeGenFunction::getNonTrivialDestructorStr(QualType QT, CharUnits Alignment,
+                                            bool IsVolatile, ASTContext &Ctx) {
+  GenDestructorFuncName GenName("", Alignment, Ctx);
+  return GenName.getName(QT, IsVolatile);
+}
+
 void CodeGenFunction::callCStructDestructor(LValue Dst) {
   bool IsVolatile = Dst.isVolatile();
   Address DstPtr = Dst.getAddress();
   QualType QT = Dst.getType();
-  GenDestructorFuncName GenName(DstPtr.getAlignment(), getContext());
+  GenDestructorFuncName GenName("__destructor_", DstPtr.getAlignment(),
+                                getContext());
   std::string FuncName = GenName.getName(QT, IsVolatile);
   callSpecialFunction(GenDestructor(getContext()), FuncName, QT, IsVolatile,
                       *this, std::array<Address, 1>({{DstPtr}}));
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 98975a8c752..030101d2f4e 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -772,9 +772,11 @@ static bool endsWithReturn(const Decl* F) {
   return false;
 }
 
-static void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
-  Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
-  Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
+void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
+  if (SanOpts.has(SanitizerKind::Thread)) {
+    Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
+    Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
+  }
 }
 
 static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
@@ -887,10 +889,6 @@ void CodeGenFunction::StartFunction(GlobalDecl GD,
           (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) {
         markAsIgnoreThreadCheckingAtRuntime(Fn);
       }
-    } else if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D)) {
-      IdentifierInfo *II = FD->getIdentifier();
-      if (II && II->isStr("__destroy_helper_block_"))
-        markAsIgnoreThreadCheckingAtRuntime(Fn);
     }
   }
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 878923a85bd..a160c368b19 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1746,6 +1746,9 @@ public:
                                         bool IsLambdaConversionToBlock,
                                         bool BuildGlobalBlock);
 
+  /// Check if \p T is a C++ class that has a destructor that can throw.
+  static bool cxxDestructorCanThrow(QualType T);
+
   llvm::Constant *GenerateCopyHelperFunction(const CGBlockInfo &blockInfo);
   llvm::Constant *GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo);
   llvm::Constant *GenerateObjCAtomicSetterCopyHelperFunction(
@@ -1754,7 +1757,8 @@ public:
                                              const ObjCPropertyImplDecl *PID);
   llvm::Value *EmitBlockCopyAndAutorelease(llvm::Value *Block, QualType Ty);
 
-  void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags);
+  void BuildBlockRelease(llvm::Value *DeclPtr, BlockFieldFlags flags,
+                         bool CanThrow);
 
   class AutoVarEmission;
 
@@ -1777,7 +1781,7 @@ public:
   /// \param LoadBlockVarAddr Indicates whether we need to emit a load from
   /// \p Addr to get the address of the __block structure.
   void enterByrefCleanup(CleanupKind Kind, Address Addr, BlockFieldFlags Flags,
-                         bool LoadBlockVarAddr);
+                         bool LoadBlockVarAddr, bool CanThrow);
 
   void setBlockContextParameter(const ImplicitParamDecl *D, unsigned argNum,
                                 llvm::Value *ptr);
@@ -1800,6 +1804,11 @@ public:
 
   void GenerateCode(GlobalDecl GD, llvm::Function *Fn,
                     const CGFunctionInfo &FnInfo);
+
+  /// Annotate the function with an attribute that disables TSan checking at
+  /// runtime.
+  void markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn);
+
   /// Emit code for the start of a function.
   /// \param Loc       The location to be associated with the function.
   /// \param StartLoc  The location of the function body.
@@ -3605,6 +3614,19 @@ public:
                                                CXXDtorType Type,
                                                const CXXRecordDecl *RD);
 
+  // Return the copy constructor name with the prefix "__copy_constructor_"
+  // removed.
+  static std::string getNonTrivialCopyConstructorStr(QualType QT,
+                                                     CharUnits Alignment,
+                                                     bool IsVolatile,
+                                                     ASTContext &Ctx);
+
+  // Return the destructor name with the prefix "__destructor_" removed.
+  static std::string getNonTrivialDestructorStr(QualType QT,
+                                                CharUnits Alignment,
+                                                bool IsVolatile,
+                                                ASTContext &Ctx);
+
   // These functions emit calls to the special functions of non-trivial C
   // structs.
   void defaultInitNonTrivialCStructVar(LValue Dst);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index bc966891071..abdeba60be7 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -11812,8 +11812,16 @@ void Sema::CheckCompleteVariableDeclaration(VarDecl *var) {
       if (!result.isInvalid()) {
         result = MaybeCreateExprWithCleanups(result);
         Expr *init = result.getAs<Expr>();
-        Context.setBlockVarCopyInits(var, init);
+        Context.setBlockVarCopyInit(var, init, canThrow(init));
       }
+
+      // The destructor's exception spefication is needed when IRGen generates
+      // block copy/destroy functions. Resolve it here.
+      if (const CXXRecordDecl *RD = type->getAsCXXRecordDecl())
+        if (CXXDestructorDecl *DD = RD->getDestructor()) {
+          auto *FPT = DD->getType()->getAs<FunctionProtoType>();
+          FPT = ResolveExceptionSpec(poi, FPT);
+        }
     }
   }
 
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index c8ce1a1fcdd..2b8cfac88d9 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -1381,6 +1381,12 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) {
     }
   }
 
+  if (VD->hasAttr<BlocksAttr>() && VD->getType()->getAsCXXRecordDecl()) {
+    Expr *CopyExpr = Record.readExpr();
+    if (CopyExpr)
+      Reader.getContext().setBlockVarCopyInit(VD, CopyExpr, Record.readInt());
+  }
+
   if (VD->getStorageDuration() == SD_Static && Record.readInt())
     Reader.DefinitionSource[VD] = Loc.F->Kind == ModuleKind::MK_MainFile;
 
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 4633a9c18d1..2d90c98faaa 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -946,6 +946,13 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
     Record.push_back(0);
   }
 
+  if (D->hasAttr<BlocksAttr>() && D->getType()->getAsCXXRecordDecl()) {
+    ASTContext::BlockVarCopyInit Init = Writer.Context->getBlockVarCopyInit(D);
+    Record.AddStmt(Init.getCopyExpr());
+    if (Init.getCopyExpr())
+      Record.push_back(Init.canThrow());
+  }
+
   if (D->getStorageDuration() == SD_Static) {
     bool ModulesCodegen = false;
     if (Writer.WritingModule &&
@@ -998,6 +1005,7 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
       !D->isConstexpr() &&
       !D->isInitCapture() &&
       !D->isPreviousDeclInSameBlockScope() &&
+      !(D->hasAttr<BlocksAttr>() && D->getType()->getAsCXXRecordDecl()) &&
       D->getStorageDuration() != SD_Static &&
       !D->getMemberSpecializationInfo())
     AbbrevToUse = Writer.getDeclVarAbbrev();