Use branch funnels for virtual calls when retpoline mitigation is enabled.

The retpoline mitigation for variant 2 of CVE-2017-5715 inhibits the
branch predictor, and as a result it can lead to a measurable loss of
performance. We can reduce the performance impact of retpolined virtual
calls by replacing them with a special construct known as a branch
funnel, which is an instruction sequence that implements virtual calls
to a set of known targets using a binary tree of direct branches. This
allows the processor to speculately execute valid implementations of the
virtual function without allowing for speculative execution of of calls
to arbitrary addresses.

This patch extends the whole-program devirtualization pass to replace
certain virtual calls with calls to branch funnels, which are
represented using a new llvm.icall.jumptable intrinsic. It also extends
the LowerTypeTests pass to recognize the new intrinsic, generate code
for the branch funnels (x86_64 only for now) and lay out virtual tables
as required for each branch funnel.

The implementation supports full LTO as well as ThinLTO, and extends the
ThinLTO summary format used for whole-program devirtualization to
support branch funnels.

For more details see RFC:
http://lists.llvm.org/pipermail/llvm-dev/2018-January/120672.html

Differential Revision: https://reviews.llvm.org/D42453

llvm-svn: 327163

6 files changed, 425 insertions, 12 deletions

diff --git a/llvm/test/CodeGen/X86/icall-branch-funnel.ll b/llvm/test/CodeGen/X86/icall-branch-funnel.ll
new file mode 100644
index 00000000000..010734cd856
--- /dev/null
+++ b/llvm/test/CodeGen/X86/icall-branch-funnel.ll
@@ -0,0 +1,170 @@
+; RUN: llc -mtriple=x86_64-unknown-linux < %s | FileCheck %s
+
+@g = external global i8
+
+declare void @f0()
+declare void @f1()
+declare void @f2()
+declare void @f3()
+declare void @f4()
+declare void @f5()
+declare void @f6()
+declare void @f7()
+declare void @f8()
+declare void @f9()
+
+declare void @llvm.icall.branch.funnel(...)
+
+define void @jt2(i8* nest, ...) {
+  ; CHECK: jt2:
+  ; CHECK:      leaq g+1(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB0_1
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f0
+  ; CHECK-NEXT: .LBB0_1:
+  ; CHECK-NEXT: jmp f1
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      i8* getelementptr (i8, i8* @g, i64 0), void ()* @f0,
+      i8* getelementptr (i8, i8* @g, i64 1), void ()* @f1,
+      ...
+  )
+  ret void
+}
+
+define void @jt3(i8* nest, ...) {
+  ; CHECK: jt3:
+  ; CHECK:      leaq g+1(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB1_1
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f0
+  ; CHECK-NEXT: .LBB1_1:
+  ; CHECK-NEXT: jne .LBB1_2
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f1
+  ; CHECK-NEXT: .LBB1_2:
+  ; CHECK-NEXT: jmp f2
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      i8* getelementptr (i8, i8* @g, i64 0), void ()* @f0,
+      i8* getelementptr (i8, i8* @g, i64 2), void ()* @f2,
+      i8* getelementptr (i8, i8* @g, i64 1), void ()* @f1,
+      ...
+  )
+  ret void
+}
+
+define void @jt7(i8* nest, ...) {
+  ; CHECK: jt7:
+  ; CHECK:      leaq g+3(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB2_1
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: leaq g+1(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB2_6
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f0
+  ; CHECK-NEXT: .LBB2_1:
+  ; CHECK-NEXT: jne .LBB2_2
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f3
+  ; CHECK-NEXT: .LBB2_6:
+  ; CHECK-NEXT: jne .LBB2_7
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f1
+  ; CHECK-NEXT: .LBB2_2:
+  ; CHECK-NEXT: leaq g+5(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB2_3
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f4
+  ; CHECK-NEXT: .LBB2_7:
+  ; CHECK-NEXT: jmp f2
+  ; CHECK-NEXT: .LBB2_3:
+  ; CHECK-NEXT: jne .LBB2_4
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f5
+  ; CHECK-NEXT: .LBB2_4:
+  ; CHECK-NEXT: jmp f6
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      i8* getelementptr (i8, i8* @g, i64 0), void ()* @f0,
+      i8* getelementptr (i8, i8* @g, i64 1), void ()* @f1,
+      i8* getelementptr (i8, i8* @g, i64 2), void ()* @f2,
+      i8* getelementptr (i8, i8* @g, i64 3), void ()* @f3,
+      i8* getelementptr (i8, i8* @g, i64 4), void ()* @f4,
+      i8* getelementptr (i8, i8* @g, i64 5), void ()* @f5,
+      i8* getelementptr (i8, i8* @g, i64 6), void ()* @f6,
+      ...
+  )
+  ret void
+}
+
+define void @jt10(i8* nest, ...) {
+  ; CHECK: jt10:
+  ; CHECK:      leaq g+5(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB3_1
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: leaq g+1(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB3_7
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f0
+  ; CHECK-NEXT: .LBB3_1:
+  ; CHECK-NEXT: jne .LBB3_2
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f5
+  ; CHECK-NEXT: .LBB3_7:
+  ; CHECK-NEXT: jne .LBB3_8
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f1
+  ; CHECK-NEXT: .LBB3_2:
+  ; CHECK-NEXT: leaq g+7(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB3_3
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f6
+  ; CHECK-NEXT: .LBB3_8:
+  ; CHECK-NEXT: leaq g+3(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB3_9
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f2
+  ; CHECK-NEXT: .LBB3_3:
+  ; CHECK-NEXT: jne .LBB3_4
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f7
+  ; CHECK-NEXT: .LBB3_9:
+  ; CHECK-NEXT: jne .LBB3_10
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f3
+  ; CHECK-NEXT: .LBB3_4:
+  ; CHECK-NEXT: leaq g+9(%rip), %r11
+  ; CHECK-NEXT: cmpq %r11, %r10
+  ; CHECK-NEXT: jae .LBB3_5
+  ; CHECK-NEXT: #
+  ; CHECK-NEXT: jmp f8
+  ; CHECK-NEXT: .LBB3_10:
+  ; CHECK-NEXT: jmp f4
+  ; CHECK-NEXT: .LBB3_5:
+  ; CHECK-NEXT: jmp f9
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      i8* getelementptr (i8, i8* @g, i64 0), void ()* @f0,
+      i8* getelementptr (i8, i8* @g, i64 1), void ()* @f1,
+      i8* getelementptr (i8, i8* @g, i64 2), void ()* @f2,
+      i8* getelementptr (i8, i8* @g, i64 3), void ()* @f3,
+      i8* getelementptr (i8, i8* @g, i64 4), void ()* @f4,
+      i8* getelementptr (i8, i8* @g, i64 5), void ()* @f5,
+      i8* getelementptr (i8, i8* @g, i64 6), void ()* @f6,
+      i8* getelementptr (i8, i8* @g, i64 7), void ()* @f7,
+      i8* getelementptr (i8, i8* @g, i64 8), void ()* @f8,
+      i8* getelementptr (i8, i8* @g, i64 9), void ()* @f9,
+      ...
+  )
+  ret void
+}
diff --git a/llvm/test/Transforms/LowerTypeTests/icall-branch-funnel.ll b/llvm/test/Transforms/LowerTypeTests/icall-branch-funnel.ll
new file mode 100644
index 00000000000..6cd81f275ba
--- /dev/null
+++ b/llvm/test/Transforms/LowerTypeTests/icall-branch-funnel.ll
@@ -0,0 +1,46 @@
+; RUN: opt -S -lowertypetests < %s | FileCheck %s
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux"
+
+; CHECK: @0 = private constant { i32, [0 x i8], i32 } { i32 1, [0 x i8] zeroinitializer, i32 2 }
+; CHECK: @f1 = alias void (), void ()* @.cfi.jumptable
+; CHECK: @f2 = alias void (), bitcast ([8 x i8]* getelementptr inbounds ([2 x [8 x i8]], [2 x [8 x i8]]* bitcast (void ()* @.cfi.jumptable to [2 x [8 x i8]]*), i64 0, i64 1) to void ()*)
+; CHECK: @g1 = alias i32, getelementptr inbounds ({ i32, [0 x i8], i32 }, { i32, [0 x i8], i32 }* @0, i32 0, i32 0)
+; CHECK: @g2 = alias i32, getelementptr inbounds ({ i32, [0 x i8], i32 }, { i32, [0 x i8], i32 }* @0, i32 0, i32 2)
+
+@g1 = constant i32 1
+@g2 = constant i32 2
+
+define void @f1() {
+  ret void
+}
+
+define void @f2() {
+  ret void
+}
+
+declare void @g1f()
+declare void @g2f()
+
+define void @jt2(i8* nest, ...) {
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      i32* @g1, void ()* @g1f,
+      i32* @g2, void ()* @g2f,
+      ...
+  )
+  ret void
+}
+
+define void @jt3(i8* nest, ...) {
+  musttail call void (...) @llvm.icall.branch.funnel(
+      i8* %0,
+      void ()* @f1, void ()* @f1,
+      void ()* @f2, void ()* @f2,
+      ...
+  )
+  ret void
+}
+
+declare void @llvm.icall.branch.funnel(...)
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-branch-funnel.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-branch-funnel.yaml
new file mode 100644
index 00000000000..eaa23b406bf
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-branch-funnel.yaml
@@ -0,0 +1,11 @@
+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: BranchFunnel
+  typeid2:
+    WPDRes:
+      8:
+        Kind: BranchFunnel
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp-branch-funnel.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp-branch-funnel.yaml
new file mode 100644
index 00000000000..e1b1bdbb818
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/import-vcp-branch-funnel.yaml
@@ -0,0 +1,23 @@
+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: BranchFunnel
+        ResByArg:
+          1:
+            Kind: VirtualConstProp
+            Info: 0
+            Byte: 42
+            Bit: 0
+  typeid2:
+    WPDRes:
+      8:
+        Kind: BranchFunnel
+        ResByArg:
+          3:
+            Kind: VirtualConstProp
+            Info: 0
+            Byte: 43
+            Bit: 128
+...
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
new file mode 100644
index 00000000000..3b971f04d59
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
@@ -0,0 +1,155 @@
+; RUN: opt -S -wholeprogramdevirt %s | FileCheck --check-prefixes=CHECK,RETP %s
+; RUN: sed -e 's,+retpoline,-retpoline,g' %s | opt -S -wholeprogramdevirt | FileCheck --check-prefixes=CHECK,NORETP %s
+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck --check-prefixes=CHECK,RETP %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+; SUMMARY:      TypeIdMap:       
+; SUMMARY-NEXT:   typeid1:         
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            BranchFunnel
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
+; SUMMARY-NEXT:   typeid2:         
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
+; SUMMARY-NEXT:   typeid3:         
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            BranchFunnel
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+@vt1_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1_1 to i8*)], !type !0
+@vt1_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf1_2 to i8*)], !type !0
+
+declare i32 @vf1_1(i8* %this, i32 %arg)
+declare i32 @vf1_2(i8* %this, i32 %arg)
+
+@vt2_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_1 to i8*)], !type !1
+@vt2_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_2 to i8*)], !type !1
+@vt2_3 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_3 to i8*)], !type !1
+@vt2_4 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_4 to i8*)], !type !1
+@vt2_5 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_5 to i8*)], !type !1
+@vt2_6 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_6 to i8*)], !type !1
+@vt2_7 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_7 to i8*)], !type !1
+@vt2_8 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_8 to i8*)], !type !1
+@vt2_9 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_9 to i8*)], !type !1
+@vt2_10 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_10 to i8*)], !type !1
+@vt2_11 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf2_11 to i8*)], !type !1
+
+declare i32 @vf2_1(i8* %this, i32 %arg)
+declare i32 @vf2_2(i8* %this, i32 %arg)
+declare i32 @vf2_3(i8* %this, i32 %arg)
+declare i32 @vf2_4(i8* %this, i32 %arg)
+declare i32 @vf2_5(i8* %this, i32 %arg)
+declare i32 @vf2_6(i8* %this, i32 %arg)
+declare i32 @vf2_7(i8* %this, i32 %arg)
+declare i32 @vf2_8(i8* %this, i32 %arg)
+declare i32 @vf2_9(i8* %this, i32 %arg)
+declare i32 @vf2_10(i8* %this, i32 %arg)
+declare i32 @vf2_11(i8* %this, i32 %arg)
+
+@vt3_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf3_1 to i8*)], !type !2
+@vt3_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf3_2 to i8*)], !type !2
+
+declare i32 @vf3_1(i8* %this, i32 %arg)
+declare i32 @vf3_2(i8* %this, i32 %arg)
+
+@vt4_1 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf4_1 to i8*)], !type !3
+@vt4_2 = constant [1 x i8*] [i8* bitcast (i32 (i8*, i32)* @vf4_2 to i8*)], !type !3
+
+declare i32 @vf4_1(i8* %this, i32 %arg)
+declare i32 @vf4_2(i8* %this, i32 %arg)
+
+; CHECK: define i32 @fn1
+define i32 @fn1(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; RETP: {{.*}} = bitcast {{.*}} to i8*
+  ; RETP: [[VT1:%.*]] = bitcast {{.*}} to i8*
+  ; RETP: call i32 bitcast (void (i8*, ...)* @__typeid_typeid1_0_branch_funnel to i32 (i8*, i8*, i32)*)(i8* nest [[VT1]], i8* %obj, i32 1)
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK: define i32 @fn2
+define i32 @fn2(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; CHECK: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+; CHECK: define i32 @fn3
+define i32 @fn3(i8* %obj) #0 {
+  %vtableptr = bitcast i8* %obj to [1 x i8*]**
+  %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
+  %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !4)
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr [1 x i8*], [1 x i8*]* %vtable, i32 0, i32 0
+  %fptr = load i8*, i8** %fptrptr
+  %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; RETP: call i32 bitcast (void (i8*, ...)* @branch_funnel to
+  ; NORETP: call i32 %
+  %result = call i32 %fptr_casted(i8* %obj, i32 1)
+  ret i32 %result
+}
+
+; CHECK: define internal void @branch_funnel(i8* nest, ...)
+
+; CHECK: define hidden void @__typeid_typeid1_0_branch_funnel(i8* nest, ...)
+; CHECK-NEXT: call void (...) @llvm.icall.branch.funnel(i8* %0, i8* bitcast ([1 x i8*]* @vt1_1 to i8*), i32 (i8*, i32)* @vf1_1, i8* bitcast ([1 x i8*]* @vt1_2 to i8*), i32 (i8*, i32)* @vf1_2, ...)
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i32 0, !"typeid1"}
+!1 = !{i32 0, !"typeid2"}
+!2 = !{i32 0, !"typeid3"}
+!3 = !{i32 0, !4}
+!4 = distinct !{}
+
+attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/import.ll b/llvm/test/Transforms/WholeProgramDevirt/import.ll
index 27ed286335e..525d88cb662 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/import.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/import.ll
@@ -1,10 +1,12 @@
 ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-single-impl.yaml < %s | FileCheck --check-prefixes=CHECK,SINGLE-IMPL %s
-; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-uniform-ret-val.yaml < %s | FileCheck --check-prefixes=CHECK,UNIFORM-RET-VAL %s
-; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val0.yaml < %s | FileCheck --check-prefixes=CHECK,UNIQUE-RET-VAL0 %s
-; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val1.yaml < %s | FileCheck --check-prefixes=CHECK,UNIQUE-RET-VAL1 %s
-; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP64 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-uniform-ret-val.yaml < %s | FileCheck --check-prefixes=CHECK,INDIR,UNIFORM-RET-VAL %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val0.yaml < %s | FileCheck --check-prefixes=CHECK,INDIR,UNIQUE-RET-VAL0 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val1.yaml < %s | FileCheck --check-prefixes=CHECK,INDIR,UNIQUE-RET-VAL1 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP64,INDIR %s
 ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml -mtriple=i686-unknown-linux -data-layout=e-p:32:32 < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP32 %s
 ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml -mtriple=armv7-unknown-linux -data-layout=e-p:32:32 < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-ARM %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp-branch-funnel.yaml < %s | FileCheck --check-prefixes=CHECK,VCP,VCP-X86,VCP64,BRANCH-FUNNEL %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-branch-funnel.yaml < %s | FileCheck --check-prefixes=CHECK,BRANCH-FUNNEL,BRANCH-FUNNEL-NOVCP %s
 
 target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
@@ -18,7 +20,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; constant propagation.
 
 ; CHECK: define i32 @call1
-define i32 @call1(i8* %obj) {
+define i32 @call1(i8* %obj) #0 {
   %vtableptr = bitcast i8* %obj to [3 x i8*]**
   %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
   %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
@@ -27,16 +29,18 @@ define i32 @call1(i8* %obj) {
   %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
   %fptr = load i8*, i8** %fptrptr
   %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
+  ; CHECK: {{.*}} = bitcast {{.*}} to i8*
+  ; VCP: [[VT1:%.*]] = bitcast {{.*}} to i8*
   ; SINGLE-IMPL: call i32 bitcast (void ()* @singleimpl1 to i32 (i8*, i32)*)
   %result = call i32 %fptr_casted(i8* %obj, i32 1)
   ; UNIFORM-RET-VAL: ret i32 42
-  ; VCP: {{.*}} = bitcast {{.*}} to i8*
-  ; VCP: [[VT1:%.*]] = bitcast {{.*}} to i8*
   ; VCP-X86: [[GEP1:%.*]] = getelementptr i8, i8* [[VT1]], i32 ptrtoint (i8* @__typeid_typeid1_0_1_byte to i32)
   ; VCP-ARM: [[GEP1:%.*]] = getelementptr i8, i8* [[VT1]], i32 42
   ; VCP: [[BC1:%.*]] = bitcast i8* [[GEP1]] to i32*
   ; VCP: [[LOAD1:%.*]] = load i32, i32* [[BC1]]
   ; VCP: ret i32 [[LOAD1]]
+  ; BRANCH-FUNNEL-NOVCP: [[VT1:%.*]] = bitcast {{.*}} to i8*
+  ; BRANCH-FUNNEL-NOVCP: call i32 bitcast (void ()* @__typeid_typeid1_0_branch_funnel to i32 (i8*, i8*, i32)*)(i8* nest [[VT1]], i8* %obj, i32 1)
   ret i32 %result
 }
 
@@ -44,7 +48,8 @@ define i32 @call1(i8* %obj) {
 ; constant propagation.
 
 ; CHECK: define i1 @call2
-define i1 @call2(i8* %obj) {
+define i1 @call2(i8* %obj) #0 {
+  ; BRANCH-FUNNEL: [[VT1:%.*]] = bitcast {{.*}} to i8*
   %vtableptr = bitcast i8* %obj to [1 x i8*]**
   %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
   %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
@@ -57,9 +62,8 @@ define i1 @call2(i8* %obj) {
 cont:
   %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
   ; SINGLE-IMPL: call i1 bitcast (void ()* @singleimpl2 to i1 (i8*, i32)*)
-  ; UNIFORM-RET-VAL: call i1 %
-  ; UNIQUE-RET-VAL0: call i1 %
-  ; UNIQUE-RET-VAL1: call i1 %
+  ; INDIR: call i1 %
+  ; BRANCH-FUNNEL: call i1 bitcast (void ()* @__typeid_typeid2_8_branch_funnel to i1 (i8*, i8*, i32)*)(i8* nest [[VT1]], i8* %obj, i32 undef)
   %result = call i1 %fptr_casted(i8* %obj, i32 undef)
   ret i1 %result
 
@@ -69,7 +73,7 @@ trap:
 }
 
 ; CHECK: define i1 @call3
-define i1 @call3(i8* %obj) {
+define i1 @call3(i8* %obj) #0 {
   %vtableptr = bitcast i8* %obj to [1 x i8*]**
   %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr
   %vtablei8 = bitcast [1 x i8*]* %vtable to i8*
@@ -91,6 +95,8 @@ cont:
   ; VCP-ARM: [[AND2:%.*]] = and i8 [[LOAD2]], -128
   ; VCP: [[ICMP2:%.*]] = icmp ne i8 [[AND2]], 0
   ; VCP: ret i1 [[ICMP2]]
+  ; BRANCH-FUNNEL-NOVCP: [[VT2:%.*]] = bitcast {{.*}} to i8*
+  ; BRANCH-FUNNEL-NOVCP: call i1 bitcast (void ()* @__typeid_typeid2_8_branch_funnel to i1 (i8*, i8*, i32)*)(i8* nest [[VT2]], i8* %obj, i32 3)
   ret i1 %result
 
 trap:
@@ -111,3 +117,5 @@ declare void @llvm.assume(i1)
 declare void @llvm.trap()
 declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
 declare i1 @llvm.type.test(i8*, metadata)
+
+attributes #0 = { "target-features"="+retpoline" }