Always_inline codegen rewrite.

Current implementation may end up emitting an undefined reference for
an "inline __attribute__((always_inline))" function by generating an
"available_externally alwaysinline" IR function for it and then failing to
inline all the calls. This happens when a call to such function is in dead
code. As the inliner is an SCC pass, it does not process dead code.

Libc++ relies on the compiler never emitting such undefined reference.

With this patch, we emit a pair of
1. internal alwaysinline definition (called F.alwaysinline)
2a. A stub F() { musttail call F.alwaysinline }
  -- or, depending on the linkage --
2b. A declaration of F.

The frontend ensures that F.inlinefunction is only used for direct
calls, and the stub is used for everything else (taking the address of
the function, really). Declaration (2b) is emitted in the case when
"inline" is meant for inlining only (like __gnu_inline__ and some
other cases).

This approach, among other nice properties, ensures that alwaysinline
functions are always internal, making it impossible for a direct call
to such function to produce an undefined symbol reference.

This patch is based on ideas by Chandler Carruth and Richard Smith.

llvm-svn: 247494

6 files changed, 123 insertions, 11 deletions

diff --git a/clang/lib/CodeGen/CGCXX.cpp b/clang/lib/CodeGen/CGCXX.cpp
index ad303649867..07eb91582af 100644
--- a/clang/lib/CodeGen/CGCXX.cpp
+++ b/clang/lib/CodeGen/CGCXX.cpp
@@ -109,6 +109,9 @@ bool CodeGenModule::TryEmitBaseDestructorAsAlias(const CXXDestructorDecl *D) {
       D->getType()->getAs<FunctionType>()->getCallConv())
     return true;
 
+  if (BaseD->hasAttr<AlwaysInlineAttr>())
+    return true;
+
   return TryEmitDefinitionAsAlias(GlobalDecl(D, Dtor_Base),
                                   GlobalDecl(BaseD, Dtor_Base),
                                   false);
@@ -161,14 +164,7 @@ bool CodeGenModule::TryEmitDefinitionAsAlias(GlobalDecl AliasDecl,
 
   // Instead of creating as alias to a linkonce_odr, replace all of the uses
   // of the aliasee.
-  if (llvm::GlobalValue::isDiscardableIfUnused(Linkage) &&
-     (TargetLinkage != llvm::GlobalValue::AvailableExternallyLinkage ||
-      !TargetDecl.getDecl()->hasAttr<AlwaysInlineAttr>())) {
-    // FIXME: An extern template instantiation will create functions with
-    // linkage "AvailableExternally". In libc++, some classes also define
-    // members with attribute "AlwaysInline" and expect no reference to
-    // be generated. It is desirable to reenable this optimisation after
-    // corresponding LLVM changes.
+  if (llvm::GlobalValue::isDiscardableIfUnused(Linkage)) {
     Replacements[MangledName] = Aliasee;
     return false;
   }
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index b11814f21e1..b0916ed3dcb 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -1557,7 +1557,7 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
     // -fapple-kext must inline any call to this dtor into
     // the caller's body.
     if (getLangOpts().AppleKext)
-      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
+      CGM.AddAlwaysInlineFunction(CurFn);
 
     break;
   }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index a9e7a9c5552..f85f66bd134 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -2114,7 +2114,7 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
       ".omp_task_privates_map.", &CGM.getModule());
   CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, TaskPrivatesMapFnInfo,
                                 TaskPrivatesMap);
-  TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
+  CGM.AddAlwaysInlineFunction(TaskPrivatesMap);
   CodeGenFunction CGF(CGM);
   CGF.disableDebugInfo();
   CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap,
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index bbc01cba633..4fe25ff3f31 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -448,6 +448,109 @@ void CodeGenModule::Release() {
   EmitVersionIdentMetadata();
 
   EmitTargetMetadata();
+
+  RewriteAlwaysInlineFunctions();
+}
+
+void CodeGenModule::AddAlwaysInlineFunction(llvm::Function *Fn) {
+  AlwaysInlineFunctions.push_back(Fn);
+}
+
+/// Find all uses of GV that are not direct calls or invokes.
+static void FindNonDirectCallUses(llvm::GlobalValue *GV,
+                                  llvm::SmallVectorImpl<llvm::Use *> *Uses) {
+  llvm::GlobalValue::use_iterator UI = GV->use_begin(), E = GV->use_end();
+  for (; UI != E;) {
+    llvm::Use &U = *UI;
+    ++UI;
+
+    llvm::CallSite CS(U.getUser());
+    bool isDirectCall = (CS.isCall() || CS.isInvoke()) && CS.isCallee(&U);
+    if (!isDirectCall)
+      Uses->push_back(&U);
+  }
+}
+
+/// Replace a list of uses.
+static void ReplaceUsesWith(const llvm::SmallVectorImpl<llvm::Use *> &Uses,
+                            llvm::GlobalValue *V,
+                            llvm::GlobalValue *Replacement) {
+  for (llvm::Use *U : Uses) {
+    auto *C = dyn_cast<llvm::Constant>(U->getUser());
+    if (C && !isa<llvm::GlobalValue>(C))
+      C->handleOperandChange(V, Replacement, U);
+    else
+      U->set(Replacement);
+  }
+}
+
+void CodeGenModule::RewriteAlwaysInlineFunction(llvm::Function *Fn) {
+  std::string Name = Fn->getName();
+  std::string InlineName = Name + ".alwaysinline";
+  Fn->setName(InlineName);
+
+  llvm::SmallVector<llvm::Use *, 8> NonDirectCallUses;
+  Fn->removeDeadConstantUsers();
+  FindNonDirectCallUses(Fn, &NonDirectCallUses);
+  // Do not create the wrapper if there are no non-direct call uses, and we are
+  // not required to emit an external definition.
+  if (NonDirectCallUses.empty() && Fn->isDiscardableIfUnused())
+    return;
+
+  llvm::FunctionType *FT = Fn->getFunctionType();
+  llvm::LLVMContext &Ctx = getModule().getContext();
+  llvm::Function *StubFn =
+      llvm::Function::Create(FT, Fn->getLinkage(), Name, &getModule());
+  assert(StubFn->getName() == Name && "name was uniqued!");
+
+  // Insert the stub immediately after the original function. Helps with the
+  // fragile tests, among other things.
+  StubFn->removeFromParent();
+  TheModule.getFunctionList().insertAfter(Fn, StubFn);
+
+  StubFn->copyAttributesFrom(Fn);
+  StubFn->setPersonalityFn(nullptr);
+
+  // AvailableExternally functions are replaced with a declaration.
+  // Everyone else gets a wrapper that musttail-calls the original function.
+  if (Fn->hasAvailableExternallyLinkage()) {
+    StubFn->setLinkage(llvm::GlobalValue::ExternalLinkage);
+  } else {
+    llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", StubFn);
+    std::vector<llvm::Value *> Args;
+    for (llvm::Function::arg_iterator ai = StubFn->arg_begin();
+         ai != StubFn->arg_end(); ++ai)
+      Args.push_back(&*ai);
+    llvm::CallInst *CI = llvm::CallInst::Create(Fn, Args, "", BB);
+    CI->setCallingConv(Fn->getCallingConv());
+    CI->setTailCallKind(llvm::CallInst::TCK_MustTail);
+    CI->setAttributes(Fn->getAttributes());
+    if (FT->getReturnType()->isVoidTy())
+      llvm::ReturnInst::Create(Ctx, BB);
+    else
+      llvm::ReturnInst::Create(Ctx, CI, BB);
+  }
+
+  if (Fn->hasComdat())
+    StubFn->setComdat(Fn->getComdat());
+
+  ReplaceUsesWith(NonDirectCallUses, Fn, StubFn);
+
+  // Replace all metadata uses with the stub. This is primarily to reattach
+  // DISubprogram metadata to the stub, because that's what will be emitted in
+  // the object file.
+  if (Fn->isUsedByMetadata())
+    llvm::ValueAsMetadata::handleRAUW(Fn, StubFn);
+}
+
+void CodeGenModule::RewriteAlwaysInlineFunctions() {
+  for (llvm::Function *Fn : AlwaysInlineFunctions) {
+    RewriteAlwaysInlineFunction(Fn);
+    Fn->setLinkage(llvm::GlobalValue::InternalLinkage);
+    Fn->addFnAttr(llvm::Attribute::AlwaysInline);
+    Fn->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
+    Fn->setVisibility(llvm::GlobalValue::DefaultVisibility);
+  }
 }
 
 void CodeGenModule::UpdateCompletedType(const TagDecl *TD) {
@@ -772,7 +875,7 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
              !F->getAttributes().hasAttribute(llvm::AttributeSet::FunctionIndex,
                                               llvm::Attribute::NoInline)) {
     // (noinline wins over always_inline, and we can't specify both in IR)
-    B.addAttribute(llvm::Attribute::AlwaysInline);
+    AddAlwaysInlineFunction(F);
   }
 
   if (D->hasAttr<ColdAttr>()) {
diff --git a/clang/lib/CodeGen/CodeGenModule.h b/clang/lib/CodeGen/CodeGenModule.h
index 754e3d1d29f..e579f8bd5ec 100644
--- a/clang/lib/CodeGen/CodeGenModule.h
+++ b/clang/lib/CodeGen/CodeGenModule.h
@@ -489,6 +489,8 @@ private:
   /// MDNodes.
   llvm::DenseMap<QualType, llvm::Metadata *> MetadataIdMap;
 
+  llvm::SmallVector<llvm::Function*, 8> AlwaysInlineFunctions;
+
 public:
   CodeGenModule(ASTContext &C, const HeaderSearchOptions &headersearchopts,
                 const PreprocessorOptions &ppopts,
@@ -1131,6 +1133,8 @@ public:
   /// \breif Get the declaration of std::terminate for the platform.
   llvm::Constant *getTerminateFn();
 
+  void AddAlwaysInlineFunction(llvm::Function *Fn);
+
 private:
   llvm::Constant *
   GetOrCreateLLVMFunction(StringRef MangledName, llvm::Type *Ty, GlobalDecl D,
@@ -1226,6 +1230,12 @@ private:
   /// Emits target specific Metadata for global declarations.
   void EmitTargetMetadata();
 
+  /// Replaces alwaysinline functions with a pair of internal xxx.inlinefunction
+  /// for direct calls, and a stub for indirect calls, and rewrites all uses of
+  /// those.
+  void RewriteAlwaysInlineFunctions();
+  void RewriteAlwaysInlineFunction(llvm::Function *Fn);
+
   /// Emit the llvm.gcov metadata used to tell LLVM where to emit the .gcno and
   /// .gcda files in a way that persists in .bc files.
   void EmitCoverageFile();
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 0c6a6d751c7..fa714d7ddc7 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -3311,6 +3311,9 @@ static StructorCodegen getCodegenToUse(CodeGenModule &CGM,
   if (MD->getParent()->getNumVBases())
     return StructorCodegen::Emit;
 
+  if (MD->hasAttr<AlwaysInlineAttr>())
+    return StructorCodegen::Emit;
+
   GlobalDecl AliasDecl;
   if (const auto *DD = dyn_cast<CXXDestructorDecl>(MD)) {
     AliasDecl = GlobalDecl(DD, Dtor_Complete);