1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
; RUN: opt -S -loop-predication -loop-predication-skip-profitability-checks=false < %s 2>&1 | FileCheck %s
; RUN: opt -S -loop-predication-skip-profitability-checks=false -passes='require<scalar-evolution>,require<branch-prob>,loop(loop-predication)' < %s 2>&1 | FileCheck %s
; latch block exits to a speculation block. BPI already knows (without prof
; data) that deopt is very rarely
; taken. So we do not predicate this loop using that coarse latch check.
; LatchExitProbability: 0x04000000 / 0x80000000 = 3.12%
; ExitingBlockProbability: 0x7ffa572a / 0x80000000 = 99.98%
define i64 @donot_predicate(i64* nocapture readonly %arg, i32 %length, i64* nocapture readonly %arg2, i64* nocapture readonly %n_addr, i64 %i) {
; CHECK-LABEL: donot_predicate(
entry:
%length.ext = zext i32 %length to i64
%n.pre = load i64, i64* %n_addr, align 4
br label %Header
; CHECK-LABEL: Header:
; CHECK: %within.bounds = icmp ult i64 %j2, %length.ext
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9)
Header: ; preds = %entry, %Latch
%result.in3 = phi i64* [ %arg2, %entry ], [ %arg, %Latch ]
%j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]
%within.bounds = icmp ult i64 %j2, %length.ext
call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
%innercmp = icmp eq i64 %j2, %n.pre
%j.next = add nuw nsw i64 %j2, 1
br i1 %innercmp, label %Latch, label %exit, !prof !0
Latch: ; preds = %Header
%speculate_trip_count = icmp ult i64 %j.next, 1048576
br i1 %speculate_trip_count, label %Header, label %deopt
deopt: ; preds = %Latch
%counted_speculation_failed = call i64 (...) @llvm.experimental.deoptimize.i64(i64 30) [ "deopt"(i32 0) ]
ret i64 %counted_speculation_failed
exit: ; preds = %Header
%result.in3.lcssa = phi i64* [ %result.in3, %Header ]
%result.le = load i64, i64* %result.in3.lcssa, align 8
ret i64 %result.le
}
!0 = !{!"branch_weights", i32 18, i32 104200}
; predicate loop since there's no profile information and BPI concluded all
; exiting blocks have same probability of exiting from loop.
define i64 @predicate(i64* nocapture readonly %arg, i32 %length, i64* nocapture readonly %arg2, i64* nocapture readonly %n_addr, i64 %i) {
; CHECK-LABEL: predicate(
; CHECK-LABEL: entry:
; CHECK: [[limit_check:[^ ]+]] = icmp ule i64 1048576, %length.ext
; CHECK-NEXT: [[first_iteration_check:[^ ]+]] = icmp ult i64 0, %length.ext
; CHECK-NEXT: [[wide_cond:[^ ]+]] = and i1 [[first_iteration_check]], [[limit_check]]
entry:
%length.ext = zext i32 %length to i64
%n.pre = load i64, i64* %n_addr, align 4
br label %Header
; CHECK-LABEL: Header:
; CHECK: call void (i1, ...) @llvm.experimental.guard(i1 [[wide_cond]], i32 9) [ "deopt"() ]
Header: ; preds = %entry, %Latch
%result.in3 = phi i64* [ %arg2, %entry ], [ %arg, %Latch ]
%j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]
%within.bounds = icmp ult i64 %j2, %length.ext
call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
%innercmp = icmp eq i64 %j2, %n.pre
%j.next = add nuw nsw i64 %j2, 1
br i1 %innercmp, label %Latch, label %exit
Latch: ; preds = %Header
%speculate_trip_count = icmp ult i64 %j.next, 1048576
br i1 %speculate_trip_count, label %Header, label %exitLatch
exitLatch: ; preds = %Latch
ret i64 1
exit: ; preds = %Header
%result.in3.lcssa = phi i64* [ %result.in3, %Header ]
%result.le = load i64, i64* %result.in3.lcssa, align 8
ret i64 %result.le
}
; Same as test above but with profiling data that the most probable exit from
; the loop is the header exiting block (not the latch block). So do not predicate.
; LatchExitProbability: 0x000020e1 / 0x80000000 = 0.00%
; ExitingBlockProbability: 0x7ffcbb86 / 0x80000000 = 99.99%
define i64 @donot_predicate_prof(i64* nocapture readonly %arg, i32 %length, i64* nocapture readonly %arg2, i64* nocapture readonly %n_addr, i64 %i) {
; CHECK-LABEL: donot_predicate_prof(
; CHECK-LABEL: entry:
entry:
%length.ext = zext i32 %length to i64
%n.pre = load i64, i64* %n_addr, align 4
br label %Header
; CHECK-LABEL: Header:
; CHECK: %within.bounds = icmp ult i64 %j2, %length.ext
; CHECK-NEXT: call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9)
Header: ; preds = %entry, %Latch
%result.in3 = phi i64* [ %arg2, %entry ], [ %arg, %Latch ]
%j2 = phi i64 [ 0, %entry ], [ %j.next, %Latch ]
%within.bounds = icmp ult i64 %j2, %length.ext
call void (i1, ...) @llvm.experimental.guard(i1 %within.bounds, i32 9) [ "deopt"() ]
%innercmp = icmp eq i64 %j2, %n.pre
%j.next = add nuw nsw i64 %j2, 1
br i1 %innercmp, label %Latch, label %exit, !prof !1
Latch: ; preds = %Header
%speculate_trip_count = icmp ult i64 %j.next, 1048576
br i1 %speculate_trip_count, label %Header, label %exitLatch, !prof !2
exitLatch: ; preds = %Latch
ret i64 1
exit: ; preds = %Header
%result.in3.lcssa = phi i64* [ %result.in3, %Header ]
%result.le = load i64, i64* %result.in3.lcssa, align 8
ret i64 %result.le
}
declare i64 @llvm.experimental.deoptimize.i64(...)
declare void @llvm.experimental.guard(i1, ...)
!1 = !{!"branch_weights", i32 104, i32 1042861}
!2 = !{!"branch_weights", i32 255129, i32 1}
|
