16 files changed, 426 insertions, 49 deletions
diff --git a/llvm/test/CodeGen/SystemZ/Large/branch-range-03.py b/llvm/test/CodeGen/SystemZ/Large/branch-range-03.py
new file mode 100644
index 00000000000..75cdf247c6f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/Large/branch-range-03.py
@@ -0,0 +1,107 @@
+# Test 32-bit COMPARE AND BRANCH in cases where the sheer number of
+# instructions causes some branches to be out of range.
+# RUN: python %s | llc -mtriple=s390x-linux-gnu | FileCheck %s
+
+# Construct:
+#
+# before0:
+#   conditional branch to after0
+#   ...
+# beforeN:
+#   conditional branch to after0
+# main:
+#   0xffcc bytes, from MVIY instructions
+#   conditional branch to main
+# after0:
+#   ...
+#   conditional branch to main
+# afterN:
+#
+# Each conditional branch sequence occupies 12 bytes if it uses a short
+# branch and 14 if it uses a long one.  The ones before "main:" have to
+# take the branch length into account, which is 6 for short branches,
+# so the final (0x34 - 6) / 12 == 3 blocks can use short branches.
+# The ones after "main:" do not, so the first 0x34 / 12 == 4 blocks
+# can use short branches.
+#
+# CHECK: lb [[REG:%r[0-5]]], 0(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL:\.L[^ ]*]]
+# CHECK: lb [[REG:%r[0-5]]], 1(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 2(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 3(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 4(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 5(%r3)
+# CHECK: crje %r4, [[REG]], [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 6(%r3)
+# CHECK: crje %r4, [[REG]], [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 7(%r3)
+# CHECK: crje %r4, [[REG]], [[LABEL]]
+# ...main goes here...
+# CHECK: lb [[REG:%r[0-5]]], 25(%r3)
+# CHECK: crje %r4, [[REG]], [[LABEL:\.L[^ ]*]]
+# CHECK: lb [[REG:%r[0-5]]], 26(%r3)
+# CHECK: crje %r4, [[REG]], [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 27(%r3)
+# CHECK: crje %r4, [[REG]], [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 28(%r3)
+# CHECK: crje %r4, [[REG]], [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 29(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 30(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 31(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lb [[REG:%r[0-5]]], 32(%r3)
+# CHECK: cr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+
+branch_blocks = 8
+main_size = 0xffcc
+
+print 'define void @f1(i8 *%base, i8 *%stop, i32 %limit) {'
+print 'entry:'
+print '  br label %before0'
+print ''
+
+for i in xrange(branch_blocks):
+    next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
+    print 'before%d:' % i
+    print '  %%bstop%d = getelementptr i8 *%%stop, i64 %d' % (i, i)
+    print '  %%bcur%d = load volatile i8 *%%bstop%d' % (i, i)
+    print '  %%bext%d = sext i8 %%bcur%d to i32' % (i, i)
+    print '  %%btest%d = icmp eq i32 %%limit, %%bext%d' % (i, i)
+    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
+    print ''
+
+print '%s:' % next
+a, b = 1, 1
+for i in xrange(0, main_size, 6):
+    a, b = b, a + b
+    offset = 4096 + b % 500000
+    value = a % 256
+    print '  %%ptr%d = getelementptr i8 *%%base, i64 %d' % (i, offset)
+    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+
+for i in xrange(branch_blocks):
+    print '  %%astop%d = getelementptr i8 *%%stop, i64 %d' % (i, i + 25)
+    print '  %%acur%d = load volatile i8 *%%astop%d' % (i, i)
+    print '  %%aext%d = sext i8 %%acur%d to i32' % (i, i)
+    print '  %%atest%d = icmp eq i32 %%limit, %%aext%d' % (i, i)
+    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
+    print ''
+    print 'after%d:' % i
+
+print '  ret void'
+print '}'
diff --git a/llvm/test/CodeGen/SystemZ/Large/branch-range-04.py b/llvm/test/CodeGen/SystemZ/Large/branch-range-04.py
new file mode 100644
index 00000000000..3ae3ae9c37f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/Large/branch-range-04.py
@@ -0,0 +1,111 @@
+# Test 64-bit COMPARE AND BRANCH in cases where the sheer number of
+# instructions causes some branches to be out of range.
+# RUN: python %s | llc -mtriple=s390x-linux-gnu | FileCheck %s
+
+# Construct:
+#
+# before0:
+#   conditional branch to after0
+#   ...
+# beforeN:
+#   conditional branch to after0
+# main:
+#   0xffcc bytes, from MVIY instructions
+#   conditional branch to main
+# after0:
+#   ...
+#   conditional branch to main
+# afterN:
+#
+# Each conditional branch sequence occupies 12 bytes if it uses a short
+# branch and 16 if it uses a long one.  The ones before "main:" have to
+# take the branch length into account, which is 6 for short branches,
+# so the final (0x34 - 6) / 12 == 3 blocks can use short branches.
+# The ones after "main:" do not, so the first 0x34 / 12 == 4 blocks
+# can use short branches.  The conservative algorithm we use makes
+# one of the forward branches unnecessarily long, as noted in the
+# check output below.
+#
+# CHECK: lgb [[REG:%r[0-5]]], 0(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL:\.L[^ ]*]]
+# CHECK: lgb [[REG:%r[0-5]]], 1(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 2(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 3(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 4(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# ...as mentioned above, the next one could be a CGRJE instead...
+# CHECK: lgb [[REG:%r[0-5]]], 5(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 6(%r3)
+# CHECK: cgrje %r4, [[REG]], [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 7(%r3)
+# CHECK: cgrje %r4, [[REG]], [[LABEL]]
+# ...main goes here...
+# CHECK: lgb [[REG:%r[0-5]]], 25(%r3)
+# CHECK: cgrje %r4, [[REG]], [[LABEL:\.L[^ ]*]]
+# CHECK: lgb [[REG:%r[0-5]]], 26(%r3)
+# CHECK: cgrje %r4, [[REG]], [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 27(%r3)
+# CHECK: cgrje %r4, [[REG]], [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 28(%r3)
+# CHECK: cgrje %r4, [[REG]], [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 29(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 30(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 31(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+# CHECK: lgb [[REG:%r[0-5]]], 32(%r3)
+# CHECK: cgr %r4, [[REG]]
+# CHECK: jge [[LABEL]]
+
+branch_blocks = 8
+main_size = 0xffcc
+
+print 'define void @f1(i8 *%base, i8 *%stop, i64 %limit) {'
+print 'entry:'
+print '  br label %before0'
+print ''
+
+for i in xrange(branch_blocks):
+    next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
+    print 'before%d:' % i
+    print '  %%bstop%d = getelementptr i8 *%%stop, i64 %d' % (i, i)
+    print '  %%bcur%d = load volatile i8 *%%bstop%d' % (i, i)
+    print '  %%bext%d = sext i8 %%bcur%d to i64' % (i, i)
+    print '  %%btest%d = icmp eq i64 %%limit, %%bext%d' % (i, i)
+    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
+    print ''
+
+print '%s:' % next
+a, b = 1, 1
+for i in xrange(0, main_size, 6):
+    a, b = b, a + b
+    offset = 4096 + b % 500000
+    value = a % 256
+    print '  %%ptr%d = getelementptr i8 *%%base, i64 %d' % (i, offset)
+    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+
+for i in xrange(branch_blocks):
+    print '  %%astop%d = getelementptr i8 *%%stop, i64 %d' % (i, i + 25)
+    print '  %%acur%d = load volatile i8 *%%astop%d' % (i, i)
+    print '  %%aext%d = sext i8 %%acur%d to i64' % (i, i)
+    print '  %%atest%d = icmp eq i64 %%limit, %%aext%d' % (i, i)
+    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
+    print ''
+    print 'after%d:' % i
+
+print '  ret void'
+print '}'
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll
index 83d6156c5db..bf490d89294 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-01.ll
@@ -19,8 +19,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK: l [[OLD:%r[0-9]+]], 0(%r2)
 ; CHECK: [[LOOP:\.[^:]*]]:
 ; CHECK: rll [[ROT:%r[0-9]+]], [[OLD]], 0([[SHIFT]])
-; CHECK: cr [[ROT]], %r3
-; CHECK: jle [[KEEP:\..*]]
+; CHECK: crjle [[ROT]], %r3, [[KEEP:\..*]]
 ; CHECK: risbg [[ROT]], %r3, 32, 39, 0
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
@@ -40,7 +39,7 @@ define i8 @f1(i8 *%src, i8 %b) {
 ; CHECK-SHIFT2: f1:
 ; CHECK-SHIFT2: sll %r3, 24
 ; CHECK-SHIFT2: rll
-; CHECK-SHIFT2: cr {{%r[0-9]+}}, %r3
+; CHECK-SHIFT2: crjle {{%r[0-9]+}}, %r3
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: br %r14
@@ -56,8 +55,7 @@ define i8 @f2(i8 *%src, i8 %b) {
 ; CHECK: l [[OLD:%r[0-9]+]], 0(%r2)
 ; CHECK: [[LOOP:\.[^:]*]]:
 ; CHECK: rll [[ROT:%r[0-9]+]], [[OLD]], 0([[SHIFT]])
-; CHECK: cr [[ROT]], %r3
-; CHECK: jhe [[KEEP:\..*]]
+; CHECK: crjhe [[ROT]], %r3, [[KEEP:\..*]]
 ; CHECK: risbg [[ROT]], %r3, 32, 39, 0
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
@@ -77,7 +75,7 @@ define i8 @f2(i8 *%src, i8 %b) {
 ; CHECK-SHIFT2: f2:
 ; CHECK-SHIFT2: sll %r3, 24
 ; CHECK-SHIFT2: rll
-; CHECK-SHIFT2: cr {{%r[0-9]+}}, %r3
+; CHECK-SHIFT2: crjhe {{%r[0-9]+}}, %r3
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: br %r14
@@ -164,7 +162,7 @@ define i8 @f4(i8 *%src, i8 %b) {
 define i8 @f5(i8 *%src) {
 ; CHECK: f5:
 ; CHECK: llilh [[SRC2:%r[0-9]+]], 33024
-; CHECK: cr [[ROT:%r[0-9]+]], [[SRC2]]
+; CHECK: crjle [[ROT:%r[0-9]+]], [[SRC2]]
 ; CHECK: risbg [[ROT]], [[SRC2]], 32, 39, 0
 ; CHECK: br %r14
 ;
@@ -181,7 +179,7 @@ define i8 @f5(i8 *%src) {
 define i8 @f6(i8 *%src) {
 ; CHECK: f6:
 ; CHECK: llilh [[SRC2:%r[0-9]+]], 32256
-; CHECK: cr [[ROT:%r[0-9]+]], [[SRC2]]
+; CHECK: crjhe [[ROT:%r[0-9]+]], [[SRC2]]
 ; CHECK: risbg [[ROT]], [[SRC2]], 32, 39, 0
 ; CHECK: br %r14
 ;
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll
index 27dc3e925b3..b2c7bc90288 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-02.ll
@@ -19,8 +19,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK: l [[OLD:%r[0-9]+]], 0(%r2)
 ; CHECK: [[LOOP:\.[^:]*]]:
 ; CHECK: rll [[ROT:%r[0-9]+]], [[OLD]], 0([[SHIFT]])
-; CHECK: cr [[ROT]], %r3
-; CHECK: jle [[KEEP:\..*]]
+; CHECK: crjle [[ROT]], %r3, [[KEEP:\..*]]
 ; CHECK: risbg [[ROT]], %r3, 32, 47, 0
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
@@ -40,7 +39,7 @@ define i16 @f1(i16 *%src, i16 %b) {
 ; CHECK-SHIFT2: f1:
 ; CHECK-SHIFT2: sll %r3, 16
 ; CHECK-SHIFT2: rll
-; CHECK-SHIFT2: cr {{%r[0-9]+}}, %r3
+; CHECK-SHIFT2: crjle {{%r[0-9]+}}, %r3
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: br %r14
@@ -56,8 +55,7 @@ define i16 @f2(i16 *%src, i16 %b) {
 ; CHECK: l [[OLD:%r[0-9]+]], 0(%r2)
 ; CHECK: [[LOOP:\.[^:]*]]:
 ; CHECK: rll [[ROT:%r[0-9]+]], [[OLD]], 0([[SHIFT]])
-; CHECK: cr [[ROT]], %r3
-; CHECK: jhe [[KEEP:\..*]]
+; CHECK: crjhe [[ROT]], %r3, [[KEEP:\..*]]
 ; CHECK: risbg [[ROT]], %r3, 32, 47, 0
 ; CHECK: [[KEEP]]:
 ; CHECK: rll [[NEW:%r[0-9]+]], [[ROT]], 0({{%r[1-9]+}})
@@ -77,7 +75,7 @@ define i16 @f2(i16 *%src, i16 %b) {
 ; CHECK-SHIFT2: f2:
 ; CHECK-SHIFT2: sll %r3, 16
 ; CHECK-SHIFT2: rll
-; CHECK-SHIFT2: cr {{%r[0-9]+}}, %r3
+; CHECK-SHIFT2: crjhe {{%r[0-9]+}}, %r3
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: rll
 ; CHECK-SHIFT2: br %r14
@@ -164,7 +162,7 @@ define i16 @f4(i16 *%src, i16 %b) {
 define i16 @f5(i16 *%src) {
 ; CHECK: f5:
 ; CHECK: llilh [[SRC2:%r[0-9]+]], 32769
-; CHECK: cr [[ROT:%r[0-9]+]], [[SRC2]]
+; CHECK: crjle [[ROT:%r[0-9]+]], [[SRC2]]
 ; CHECK: risbg [[ROT]], [[SRC2]], 32, 47, 0
 ; CHECK: br %r14
 ;
@@ -181,7 +179,7 @@ define i16 @f5(i16 *%src) {
 define i16 @f6(i16 *%src) {
 ; CHECK: f6:
 ; CHECK: llilh [[SRC2:%r[0-9]+]], 32766
-; CHECK: cr [[ROT:%r[0-9]+]], [[SRC2]]
+; CHECK: crjhe [[ROT:%r[0-9]+]], [[SRC2]]
 ; CHECK: risbg [[ROT]], [[SRC2]], 32, 47, 0
 ; CHECK: br %r14
 ;
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll
index 53d0e737e07..4f7d820adaf 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll
@@ -7,9 +7,8 @@ define i32 @f1(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: f1:
 ; CHECK: l %r2, 0(%r3)
 ; CHECK: [[LOOP:\.[^:]*]]:
-; CHECK: cr %r2, %r4
 ; CHECK: lr [[NEW:%r[0-9]+]], %r2
-; CHECK: jle [[KEEP:\..*]]
+; CHECK: crjle %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lr [[NEW]], %r4
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
 ; CHECK: jlh [[LOOP]]
@@ -23,9 +22,8 @@ define i32 @f2(i32 %dummy, i32 *%src, i32 %b) {
 ; CHECK: f2:
 ; CHECK: l %r2, 0(%r3)
 ; CHECK: [[LOOP:\.[^:]*]]:
-; CHECK: cr %r2, %r4
 ; CHECK: lr [[NEW:%r[0-9]+]], %r2
-; CHECK: jhe [[KEEP:\..*]]
+; CHECK: crjhe %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lr [[NEW]], %r4
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
 ; CHECK: jlh [[LOOP]]
@@ -164,9 +162,8 @@ define i32 @f13(i32 %dummy, i32 *%ptr) {
 ; CHECK: lhi [[LIMIT:%r[0-9]+]], 42
 ; CHECK: l %r2, 0(%r3)
 ; CHECK: [[LOOP:\.[^:]*]]:
-; CHECK: cr %r2, [[LIMIT]]
 ; CHECK: lr [[NEW:%r[0-9]+]], %r2
-; CHECK: jle [[KEEP:\..*]]
+; CHECK: crjle %r2, [[LIMIT]], [[KEEP:\..*]]
 ; CHECK: lr [[NEW]], [[LIMIT]]
 ; CHECK: cs %r2, [[NEW]], 0(%r3)
 ; CHECK: jlh [[LOOP]]
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll
index d7723314180..cd35ab019e0 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll
@@ -7,9 +7,8 @@ define i64 @f1(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: f1:
 ; CHECK: lg %r2, 0(%r3)
 ; CHECK: [[LOOP:\.[^:]*]]:
-; CHECK: cgr %r2, %r4
 ; CHECK: lgr [[NEW:%r[0-9]+]], %r2
-; CHECK: jle [[KEEP:\..*]]
+; CHECK: cgrjle %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lgr [[NEW]], %r4
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
 ; CHECK: jlh [[LOOP]]
@@ -23,9 +22,8 @@ define i64 @f2(i64 %dummy, i64 *%src, i64 %b) {
 ; CHECK: f2:
 ; CHECK: lg %r2, 0(%r3)
 ; CHECK: [[LOOP:\.[^:]*]]:
-; CHECK: cgr %r2, %r4
 ; CHECK: lgr [[NEW:%r[0-9]+]], %r2
-; CHECK: jhe [[KEEP:\..*]]
+; CHECK: cgrjhe %r2, %r4, [[KEEP:\..*]]
 ; CHECK: lgr [[NEW]], %r4
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
 ; CHECK: jlh [[LOOP]]
@@ -131,9 +129,8 @@ define i64 @f10(i64 %dummy, i64 *%ptr) {
 ; CHECK: lghi [[LIMIT:%r[0-9]+]], 42
 ; CHECK: lg %r2, 0(%r3)
 ; CHECK: [[LOOP:\.[^:]*]]:
-; CHECK: cgr %r2, [[LIMIT]]
 ; CHECK: lgr [[NEW:%r[0-9]+]], %r2
-; CHECK: jle [[KEEP:\..*]]
+; CHECK: cgrjle %r2, [[LIMIT]], [[KEEP:\..*]]
 ; CHECK: lgr [[NEW]], [[LIMIT]]
 ; CHECK: csg %r2, [[NEW]], 0(%r3)
 ; CHECK: jlh [[LOOP]]
diff --git a/llvm/test/CodeGen/SystemZ/branch-02.ll b/llvm/test/CodeGen/SystemZ/branch-02.ll
index 9365f161430..9f71c053c7b 100644
--- a/llvm/test/CodeGen/SystemZ/branch-02.ll
+++ b/llvm/test/CodeGen/SystemZ/branch-02.ll
@@ -1,5 +1,6 @@
 ; Test all condition-code masks that are relevant for signed integer
-; comparisons.
+; comparisons, in cases where a separate branch is better than COMPARE
+; AND BRANCH.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
diff --git a/llvm/test/CodeGen/SystemZ/branch-06.ll b/llvm/test/CodeGen/SystemZ/branch-06.ll
new file mode 100644
index 00000000000..3854045e38f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/branch-06.ll
@@ -0,0 +1,89 @@
+; Test all condition-code masks that are relevant for CRJ.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare i32 @foo();
+
+define void @f1(i32 %target) {
+; CHECK: f1:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: crje %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i32 @foo()
+  %cond = icmp eq i32 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f2(i32 %target) {
+; CHECK: f2:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: crjlh %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i32 @foo()
+  %cond = icmp ne i32 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f3(i32 %target) {
+; CHECK: f3:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: crjle %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i32 @foo()
+  %cond = icmp sle i32 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f4(i32 %target) {
+; CHECK: f4:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: crjl %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i32 @foo()
+  %cond = icmp slt i32 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f5(i32 %target) {
+; CHECK: f5:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: crjh %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i32 @foo()
+  %cond = icmp sgt i32 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f6(i32 %target) {
+; CHECK: f6:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: crjhe %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i32 @foo()
+  %cond = icmp sge i32 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/branch-07.ll b/llvm/test/CodeGen/SystemZ/branch-07.ll
new file mode 100644
index 00000000000..1cab6ff28ed
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/branch-07.ll
@@ -0,0 +1,89 @@
+; Test all condition-code masks that are relevant for CGRJ.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+declare i64 @foo();
+
+define void @f1(i64 %target) {
+; CHECK: f1:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: cgrje %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i64 @foo()
+  %cond = icmp eq i64 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f2(i64 %target) {
+; CHECK: f2:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: cgrjlh %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i64 @foo()
+  %cond = icmp ne i64 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f3(i64 %target) {
+; CHECK: f3:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: cgrjle %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i64 @foo()
+  %cond = icmp sle i64 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f4(i64 %target) {
+; CHECK: f4:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: cgrjl %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i64 @foo()
+  %cond = icmp slt i64 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f5(i64 %target) {
+; CHECK: f5:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: cgrjh %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i64 @foo()
+  %cond = icmp sgt i64 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
+
+define void @f6(i64 %target) {
+; CHECK: f6:
+; CHECK: .cfi_def_cfa_offset
+; CHECK: .L[[LABEL:.*]]:
+; CHECK: cgrjhe %r2, {{%r[0-9]+}}, .L[[LABEL]]
+  br label %loop
+loop:
+  %val = call i64 @foo()
+  %cond = icmp sge i64 %val, %target
+  br i1 %cond, label %loop, label %exit
+exit:
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll b/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll
index e8488615bd1..03fabee1328 100644
--- a/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll
+++ b/llvm/test/CodeGen/SystemZ/cmpxchg-01.ll
@@ -18,8 +18,7 @@ define i8 @f1(i8 %dummy, i8 *%src, i8 %cmp, i8 %swap) {
 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
 ; CHECK-MAIN: rll %r2, [[OLD]], 8([[SHIFT]])
 ; CHECK-MAIN: risbg %r4, %r2, 32, 55, 0
-; CHECK-MAIN: cr %r2, %r4
-; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
+; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
 ; CHECK-MAIN: risbg %r5, %r2, 32, 55, 0
 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -8({{%r[1-9]+}})
 ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0(%r3)
diff --git a/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll b/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll
index 2c2f76cd7c1..b5005bb291e 100644
--- a/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll
+++ b/llvm/test/CodeGen/SystemZ/cmpxchg-02.ll
@@ -18,8 +18,7 @@ define i16 @f1(i16 %dummy, i16 *%src, i16 %cmp, i16 %swap) {
 ; CHECK-MAIN: [[LOOP:\.[^ ]*]]:
 ; CHECK-MAIN: rll %r2, [[OLD]], 16([[SHIFT]])
 ; CHECK-MAIN: risbg %r4, %r2, 32, 47, 0
-; CHECK-MAIN: cr %r2, %r4
-; CHECK-MAIN: jlh [[EXIT:\.[^ ]*]]
+; CHECK-MAIN: crjlh %r2, %r4, [[EXIT:\.[^ ]*]]
 ; CHECK-MAIN: risbg %r5, %r2, 32, 47, 0
 ; CHECK-MAIN: rll [[NEW:%r[0-9]+]], %r5, -16({{%r[1-9]+}})
 ; CHECK-MAIN: cs [[OLD]], [[NEW]], 0(%r3)
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-02.ll b/llvm/test/CodeGen/SystemZ/int-cmp-02.ll
index 262ade0b99b..b98661ed4dd 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-02.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-02.ll
@@ -5,8 +5,7 @@
 ; Check register comparison.
 define double @f1(double %a, double %b, i32 %i1, i32 %i2) {
 ; CHECK: f1:
-; CHECK: cr %r2, %r3
-; CHECK-NEXT: jl
+; CHECK: crjl %r2, %r3
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp slt i32 %i1, %i2
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-07.ll b/llvm/test/CodeGen/SystemZ/int-cmp-07.ll
index 6e626bca7f1..48ccf5cb30b 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-07.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-07.ll
@@ -5,8 +5,7 @@
 ; Check CGR.
 define double @f1(double %a, double %b, i64 %i1, i64 %i2) {
 ; CHECK: f1:
-; CHECK: cgr %r2, %r3
-; CHECK-NEXT: jl
+; CHECK: cgrjl %r2, %r3
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp slt i64 %i1, %i2
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-11.ll b/llvm/test/CodeGen/SystemZ/int-cmp-11.ll
index 876882ea8f3..64386f05d1e 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-11.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-11.ll
@@ -65,8 +65,7 @@ define double @f5(double %a, double %b, i64 %i1) {
 ; Check the next value up, which must use register comparison.
 define double @f6(double %a, double %b, i64 %i1) {
 ; CHECK: f6:
-; CHECK: cgr
-; CHECK-NEXT: jl
+; CHECK: cgrjl
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp slt i64 %i1, 2147483648
@@ -125,8 +124,7 @@ define double @f10(double %a, double %b, i64 %i1) {
 ; Check the next value down, which must use register comparison.
 define double @f11(double %a, double %b, i64 %i1) {
 ; CHECK: f11:
-; CHECK: cgr
-; CHECK-NEXT: jl
+; CHECK: cgrjl
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp slt i64 %i1, -2147483649
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-13.ll b/llvm/test/CodeGen/SystemZ/int-cmp-13.ll
index 0eec8903056..aab95473a00 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-13.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-13.ll
@@ -77,8 +77,7 @@ define double @f6(double %a, double %b, i64 %i1) {
 ; Check the next value up, which must use a register comparison.
 define double @f7(double %a, double %b, i64 %i1) {
 ; CHECK: f7:
-; CHECK: cgr %r2,
-; CHECK-NEXT: je
+; CHECK: cgrje %r2,
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp eq i64 %i1, 4294967296
@@ -137,8 +136,7 @@ define double @f11(double %a, double %b, i64 %i1) {
 ; Check the next value down, which must use register comparison.
 define double @f12(double %a, double %b, i64 %i1) {
 ; CHECK: f12:
-; CHECK: cgr
-; CHECK-NEXT: je
+; CHECK: cgrje
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp eq i64 %i1, -2147483649
diff --git a/llvm/test/CodeGen/SystemZ/int-cmp-14.ll b/llvm/test/CodeGen/SystemZ/int-cmp-14.ll
index e3a561e38aa..28c325c005e 100644
--- a/llvm/test/CodeGen/SystemZ/int-cmp-14.ll
+++ b/llvm/test/CodeGen/SystemZ/int-cmp-14.ll
@@ -77,8 +77,7 @@ define double @f6(double %a, double %b, i64 %i1) {
 ; Check the next value up, which must use a register comparison.
 define double @f7(double %a, double %b, i64 %i1) {
 ; CHECK: f7:
-; CHECK: cgr %r2,
-; CHECK-NEXT: jlh
+; CHECK: cgrjlh %r2,
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp ne i64 %i1, 4294967296
@@ -137,8 +136,7 @@ define double @f11(double %a, double %b, i64 %i1) {
 ; Check the next value down, which must use register comparison.
 define double @f12(double %a, double %b, i64 %i1) {
 ; CHECK: f12:
-; CHECK: cgr
-; CHECK-NEXT: jlh
+; CHECK: cgrjlh
 ; CHECK: ldr %f0, %f2
 ; CHECK: br %r14
   %cond = icmp ne i64 %i1, -2147483649