[GlobalOpt] don't change CC of musttail calle(e|r)

When the function has musttail call - its cc is fixed to be equal to the
cc of the musttail callee. In such case (and in the case of the musttail
callee), GlobalOpt should not change the cc to fastcc as it will break
the invariant.

This fixes PR36546

Patch by: Fedor Indutny (indutny)

Differential revision: https://reviews.llvm.org/D43859

llvm-svn: 326376

2 files changed, 58 insertions, 1 deletions

diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index b92accd2796..bdeda543e2b 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -2115,8 +2115,31 @@ static void RemoveNestAttribute(Function *F) {
 /// GHC, or anyregcc.
 static bool hasChangeableCC(Function *F) {
   CallingConv::ID CC = F->getCallingConv();
+
   // FIXME: Is it worth transforming x86_stdcallcc and x86_fastcallcc?
-  return CC == CallingConv::C || CC == CallingConv::X86_ThisCall;
+  if (CC != CallingConv::C && CC != CallingConv::X86_ThisCall)
+    return false;
+
+  // FIXME: Change CC for the whole chain of musttail calls when possible.
+  //
+  // Can't change CC of the function that either has musttail calls, or is a
+  // musttail callee itself
+  for (User *U : F->users()) {
+    if (isa<BlockAddress>(U))
+      continue;
+    CallInst* CI = dyn_cast<CallInst>(U);
+    if (!CI)
+      continue;
+
+    if (CI->isMustTailCall())
+      return false;
+  }
+
+  for (BasicBlock &BB : *F)
+    if (BB.getTerminatingMustTailCall())
+      return false;
+
+  return true;
 }
 
 /// Return true if the block containing the call site has a BlockFrequency of
diff --git a/llvm/test/Transforms/GlobalOpt/musttail_cc.ll b/llvm/test/Transforms/GlobalOpt/musttail_cc.ll
new file mode 100644
index 00000000000..fc927ea91dd
--- /dev/null
+++ b/llvm/test/Transforms/GlobalOpt/musttail_cc.ll
@@ -0,0 +1,34 @@
+; RUN: opt < %s -globalopt -S | FileCheck %s
+; PR36546
+
+; Check that musttail callee preserves its calling convention
+
+define i32 @test(i32 %a) {
+  ; CHECK: %ca = musttail call i32 @foo(i32 %a)
+  %ca = musttail call i32 @foo(i32 %a)
+  ret i32 %ca
+}
+
+; CHECK-LABEL: define internal i32 @foo(i32 %a)
+define internal i32 @foo(i32 %a) {
+  ret i32 %a
+}
+
+; Check that musttail caller preserves its calling convention
+
+define i32 @test2(i32 %a) {
+  %ca = call i32 @foo1(i32 %a)
+  ret i32 %ca
+}
+
+; CHECK-LABEL: define internal i32 @foo1(i32 %a)
+define internal i32 @foo1(i32 %a) {
+  ; CHECK: %ca = musttail call i32 @foo2(i32 %a)
+  %ca = musttail call i32 @foo2(i32 %a)
+  ret i32 %ca
+}
+
+; CHECK-LABEL: define internal i32 @foo2(i32 %a)
+define internal i32 @foo2(i32 %a) {
+  ret i32 %a
+}