Migrate some more fadd and fsub cases away from UnsafeFPMath control to utilize NoSignedZerosFPMath options control

Summary: Honoring no signed zeroes is also available as a user control through clang separately regardless of fastmath or UnsafeFPMath context, DAG guards should reflect this context.

Reviewers: spatel, arsenm, hfinkel, wristow, craig.topper

Reviewed By: spatel

Subscribers: rampitec, foad, nhaehnle, wuzish, nemanjai, jvesely, wdng, javed.absar, MaskRay, jsji

Differential Revision: https://reviews.llvm.org/D65170

llvm-svn: 367486

4 files changed, 294 insertions, 216 deletions

diff --git a/llvm/test/CodeGen/PowerPC/fma-mutate.ll b/llvm/test/CodeGen/PowerPC/fma-mutate.ll
index e03bb22617f..86fad0af248 100644
--- a/llvm/test/CodeGen/PowerPC/fma-mutate.ll
+++ b/llvm/test/CodeGen/PowerPC/fma-mutate.ll
@@ -3,19 +3,26 @@
 ; same as the FMA target register.  The second one is legal.  The third
 ; one doesn't fit the feeding-copy pattern.
 
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -enable-unsafe-fp-math -mattr=+vsx -disable-ppc-vsx-fma-mutation=false | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=+vsx -disable-ppc-vsx-fma-mutation=false | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 declare double @llvm.sqrt.f64(double)
 
-define double @foo3(double %a) nounwind {
-  %r = call double @llvm.sqrt.f64(double %a)
-  ret double %r
-
-; CHECK: @foo3
+define double @foo3_fmf(double %a) nounwind {
+; CHECK: @foo3_fmf
 ; CHECK-NOT: fmr
 ; CHECK: xsmaddmdp
 ; CHECK: xsmaddadp
+  %r = call fast double @llvm.sqrt.f64(double %a)
+  ret double %r
+}
+
+define double @foo3_safe(double %a) nounwind {
+; CHECK: @foo3_safe
+; CHECK-NOT: fmr
+; CHECK: xssqrtdp
+  %r = call double @llvm.sqrt.f64(double %a)
+  ret double %r
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
index a62ef79f509..68d53f126c1 100644
--- a/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
+++ b/llvm/test/CodeGen/PowerPC/fmf-propagation.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -mtriple=powerpc64le -debug-only=isel -o /dev/null 2>&1                        | FileCheck %s --check-prefix=FMFDEBUG
 ; RUN: llc < %s -mtriple=powerpc64le                                                           | FileCheck %s --check-prefix=FMF
 ; RUN: llc < %s -mtriple=powerpc64le -debug-only=isel -o /dev/null 2>&1 -enable-unsafe-fp-math -enable-no-nans-fp-math | FileCheck %s --check-prefix=GLOBALDEBUG
-; RUN: llc < %s -mtriple=powerpc64le -enable-unsafe-fp-math -enable-no-nans-fp-math                                    | FileCheck %s --check-prefix=GLOBAL
+; RUN: llc < %s -mtriple=powerpc64le -enable-unsafe-fp-math -enable-no-nans-fp-math -enable-no-signed-zeros-fp-math | FileCheck %s --check-prefix=GLOBAL
 
 ; Test FP transforms using instruction/node-level fast-math-flags.
 ; We're also checking debug output to verify that FMF is propagated to the newly created nodes.
diff --git a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
index 72fbe0004e1..8aaef82761d 100644
--- a/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/qpx-recipest.ll
@@ -1,45 +1,41 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q -enable-unsafe-fp-math | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck -check-prefix=CHECK-SAFE %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=a2q | FileCheck %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
 declare <4 x double> @llvm.sqrt.v4f64(<4 x double>)
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
-define <4 x double> @foo(<4 x double> %a, <4 x double> %b) nounwind {
-entry:
-  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %r = fdiv <4 x double> %a, %x
-  ret <4 x double> %r
-
-; CHECK-LABEL: @foo
+define <4 x double> @foo_fmf(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: @foo_fmf
 ; CHECK: qvfrsqrte
 ; CHECK-DAG: qvfmul
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsub
-; CHECK-DAG: qvfmadd
-; CHECK-DAG: qvfmadd
+; CHECK-DAG: qvfmsub
+; CHECK-DAG: qvfnmsub
 ; CHECK: qvfmul
 ; CHECK: qvfmul
-; CHECK: qvfmadd
+; CHECK: qvfnmsub
 ; CHECK: qvfmul
 ; CHECK: qvfmul
 ; CHECK: blr
-
-; CHECK-SAFE-LABEL: @foo
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+entry:
+  %x = call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %r = fdiv fast <4 x double> %a, %x
+  ret <4 x double> %r
 }
 
-define <4 x double> @foof(<4 x double> %a, <4 x float> %b) nounwind {
+define <4 x double> @foo_safe(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: @foo_safe
+; CHECK: fsqrt
+; CHECK: fdiv
+; CHECK: blr
 entry:
-  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %y = fpext <4 x float> %x to <4 x double>
-  %r = fdiv <4 x double> %a, %y
+  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %r = fdiv <4 x double> %a, %x
   ret <4 x double> %r
+}
 
-; CHECK-LABEL: @foof
+define <4 x double> @foof_fmf(<4 x double> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: @foof_fmf
 ; CHECK: qvfrsqrtes
 ; CHECK-DAG: qvfmuls
 ; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
@@ -49,48 +45,59 @@ entry:
 ; CHECK: qvfmuls
 ; CHECK: qvfmul
 ; CHECK: blr
-
-; CHECK-SAFE-LABEL: @foof
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+entry:
+  %x = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %y = fpext <4 x float> %x to <4 x double>
+  %r = fdiv fast <4 x double> %a, %y
+  ret <4 x double> %r
 }
 
-define <4 x float> @food(<4 x float> %a, <4 x double> %b) nounwind {
+define <4 x double> @foof_safe(<4 x double> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: @foof_safe
+; CHECK: fsqrts
+; CHECK: fdiv
+; CHECK: blr
 entry:
-  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
-  %y = fptrunc <4 x double> %x to <4 x float>
-  %r = fdiv <4 x float> %a, %y
-  ret <4 x float> %r
+  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %y = fpext <4 x float> %x to <4 x double>
+  %r = fdiv <4 x double> %a, %y
+  ret <4 x double> %r
+}
 
-; CHECK-LABEL: @food
+define <4 x float> @food_fmf(<4 x float> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: @food_fmf
 ; CHECK: qvfrsqrte
 ; CHECK-DAG: qvfmul
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsub
-; CHECK-DAG: qvfmadd
-; CHECK-DAG: qvfmadd
+; CHECK-DAG: qvfmsub
+; CHECK-DAG: qvfnmsub
 ; CHECK: qvfmul
 ; CHECK: qvfmul
-; CHECK: qvfmadd
+; CHECK: qvfnmsub
 ; CHECK: qvfmul
 ; CHECK: qvfrsp
 ; CHECK: qvfmuls
 ; CHECK: blr
-
-; CHECK-SAFE-LABEL: @food
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+entry:
+  %x = call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %y = fptrunc <4 x double> %x to <4 x float>
+  %r = fdiv fast <4 x float> %a, %y
+  ret <4 x float> %r
 }
 
-define <4 x float> @goo(<4 x float> %a, <4 x float> %b) nounwind {
+define <4 x float> @food_safe(<4 x float> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: @food_safe
+; CHECK: fsqrt
+; CHECK: fdivs
+; CHECK: blr
 entry:
-  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
-  %r = fdiv <4 x float> %a, %x
+  %x = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %b)
+  %y = fptrunc <4 x double> %x to <4 x float>
+  %r = fdiv <4 x float> %a, %y
   ret <4 x float> %r
+}
 
-; CHECK-LABEL: @goo
+define <4 x float> @goo_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: @goo_fmf
 ; CHECK: qvfrsqrtes
 ; CHECK-DAG: qvfmuls
 ; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
@@ -100,19 +107,25 @@ entry:
 ; CHECK: qvfmuls
 ; CHECK: qvfmuls
 ; CHECK: blr
-
-; CHECK-SAFE-LABEL: @goo
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+entry:
+  %x = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %r = fdiv fast <4 x float> %a, %x
+  ret <4 x float> %r
 }
 
-define <4 x double> @foo2(<4 x double> %a, <4 x double> %b) nounwind {
+define <4 x float> @goo_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: @goo_safe
+; CHECK: fsqrts
+; CHECK: fdivs
+; CHECK: blr
 entry:
-  %r = fdiv <4 x double> %a, %b
-  ret <4 x double> %r
+  %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %r = fdiv <4 x float> %a, %x
+  ret <4 x float> %r
+}
 
-; CHECK-LABEL: @foo2
+define <4 x double> @foo2_fmf(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: @foo2_fmf
 ; CHECK: qvfre
 ; CHECK: qvfnmsub
 ; CHECK: qvfmadd
@@ -120,61 +133,70 @@ entry:
 ; CHECK: qvfmadd
 ; CHECK: qvfmul
 ; CHECK: blr
-
-; CHECK-SAFE-LABEL: @foo2
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+entry:
+  %r = fdiv fast <4 x double> %a, %b
+  ret <4 x double> %r
 }
 
-define <4 x float> @goo2(<4 x float> %a, <4 x float> %b) nounwind {
-entry:
-  %r = fdiv <4 x float> %a, %b
-  ret <4 x float> %r
+define <4 x double> @foo2_safe(<4 x double> %a, <4 x double> %b) nounwind {
+; CHECK-LABEL: @foo2_safe
+; CHECK: fdiv
+; CHECK: blr
+  %r = fdiv <4 x double> %a, %b
+  ret <4 x double> %r
+}
 
-; CHECK-LABEL: @goo2
+define <4 x float> @goo2_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: @goo2_fmf
 ; CHECK: qvfres
 ; CHECK: qvfnmsubs
 ; CHECK: qvfmadds
 ; CHECK: qvfmuls
 ; CHECK: blr
-
-; CHECK-SAFE-LABEL: @goo2
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+entry:
+  %r = fdiv fast <4 x float> %a, %b
+  ret <4 x float> %r
 }
 
-define <4 x double> @foo3(<4 x double> %a) nounwind {
+define <4 x float> @goo2_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK-LABEL: @goo2_safe
+; CHECK: fdivs
+; CHECK: blr
 entry:
-  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
-  ret <4 x double> %r
+  %r = fdiv <4 x float> %a, %b
+  ret <4 x float> %r
+}
 
-; CHECK-LABEL: @foo3
+define <4 x double> @foo3_fmf(<4 x double> %a) nounwind {
+; CHECK-LABEL: @foo3_fmf
 ; CHECK: qvfrsqrte
 ; CHECK: qvfmul
-; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
-;        an qvfmadd instead of a qvfnmsub
-; CHECK-DAG: qvfmadd
+; CHECK-DAG: qvfmsub
 ; CHECK-DAG: qvfcmpeq
-; CHECK-DAG: qvfmadd
+; CHECK-DAG: qvfnmsub
 ; CHECK-DAG: qvfmul
 ; CHECK-DAG: qvfmul
-; CHECK-DAG: qvfmadd
+; CHECK-DAG: qvfnmsub
 ; CHECK-DAG: qvfmul
 ; CHECK-DAG: qvfmul
 ; CHECK: qvfsel
 ; CHECK: blr
-
-; CHECK-SAFE-LABEL: @foo3
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: blr
+entry:
+  %r = call fast <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
+  ret <4 x double> %r
 }
 
-define <4 x float> @goo3(<4 x float> %a) nounwind {
+define <4 x double> @foo3_safe(<4 x double> %a) nounwind {
+; CHECK-LABEL: @foo3_safe
+; CHECK: fsqrt
+; CHECK: blr
 entry:
-  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
+  %r = call <4 x double> @llvm.sqrt.v4f64(<4 x double> %a)
+  ret <4 x double> %r
+}
 
-; CHECK-LABEL: @goo3
+define <4 x float> @goo3_fmf(<4 x float> %a) nounwind {
+; CHECK-LABEL: @goo3_fmf
 ; CHECK: qvfrsqrtes
 ; CHECK: qvfmuls
 ; FIXME: We're currently loading two constants here (1.5 and -1.5), and using
@@ -186,9 +208,17 @@ entry:
 ; CHECK-DAG: qvfmuls
 ; CHECK: qvfsel
 ; CHECK: blr
+entry:
+  %r = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
+}
 
-; CHECK-SAFE-LABEL: @goo3
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: blr
+define <4 x float> @goo3_safe(<4 x float> %a) nounwind {
+; CHECK-LABEL: @goo3_safe
+; CHECK: fsqrts
+; CHECK: blr
+entry:
+  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/recipest.ll b/llvm/test/CodeGen/PowerPC/recipest.ll
index b7191875190..d1bf1c9e502 100644
--- a/llvm/test/CodeGen/PowerPC/recipest.ll
+++ b/llvm/test/CodeGen/PowerPC/recipest.ll
@@ -1,5 +1,4 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -enable-unsafe-fp-math -mattr=-vsx | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck -check-prefix=CHECK-SAFE %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
@@ -8,12 +7,8 @@ declare double @llvm.sqrt.f64(double)
 declare float @llvm.sqrt.f32(float)
 declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
 
-define double @foo(double %a, double %b) nounwind {
-  %x = call double @llvm.sqrt.f64(double %b)
-  %r = fdiv double %a, %x
-  ret double %r
-
-; CHECK: @foo
+define double @foo_fmf(double %a, double %b) nounwind {
+; CHECK: @foo_fmf
 ; CHECK: frsqrte
 ; CHECK: fmul
 ; CHECK-NEXT: fmadd
@@ -25,34 +20,35 @@ define double @foo(double %a, double %b) nounwind {
 ; CHECK-NEXT: fmul
 ; CHECK-NEXT: fmul
 ; CHECK: blr
-
-; CHECK-SAFE: @foo
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+  %x = call fast double @llvm.sqrt.f64(double %b)
+  %r = fdiv fast double %a, %x
+  ret double %r
 }
 
-define double @no_estimate_refinement_f64(double %a, double %b) #0 {
+define double @foo_safe(double %a, double %b) nounwind {
+; CHECK: @foo_safe
+; CHECK: fsqrt
+; CHECK: fdiv
+; CHECK: blr
   %x = call double @llvm.sqrt.f64(double %b)
   %r = fdiv double %a, %x
   ret double %r
+}
 
+define double @no_estimate_refinement_f64(double %a, double %b) #0 {
 ; CHECK-LABEL: @no_estimate_refinement_f64
 ; CHECK: frsqrte
 ; CHECK-NOT: fmadd
 ; CHECK: fmul
 ; CHECK-NOT: fmadd
 ; CHECK: blr
-}
-
-
-define double @foof(double %a, float %b) nounwind {
-  %x = call float @llvm.sqrt.f32(float %b)
-  %y = fpext float %x to double
-  %r = fdiv double %a, %y
+  %x = call fast double @llvm.sqrt.f64(double %b)
+  %r = fdiv fast double %a, %x
   ret double %r
+}
 
-; CHECK: @foof
+define double @foof_fmf(double %a, float %b) nounwind {
+; CHECK: @foof_fmf
 ; CHECK-DAG: frsqrtes
 ; CHECK: fmuls
 ; CHECK-NEXT: fmadds
@@ -60,20 +56,25 @@ define double @foof(double %a, float %b) nounwind {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: fmul
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @foof
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+  %x = call fast float @llvm.sqrt.f32(float %b)
+  %y = fpext float %x to double
+  %r = fdiv fast double %a, %y
+  ret double %r
 }
 
-define float @food(float %a, double %b) nounwind {
-  %x = call double @llvm.sqrt.f64(double %b)
-  %y = fptrunc double %x to float
-  %r = fdiv float %a, %y
-  ret float %r
+define double @foof_safe(double %a, float %b) nounwind {
+; CHECK: @foof_safe
+; CHECK: fsqrts
+; CHECK: fdiv
+; CHECK: blr
+  %x = call float @llvm.sqrt.f32(float %b)
+  %y = fpext float %x to double
+  %r = fdiv double %a, %y
+  ret double %r
+}
 
-; CHECK: @foo
+define float @food_fmf(float %a, double %b) nounwind {
+; CHECK: @food_fmf
 ; CHECK-DAG: frsqrte
 ; CHECK: fmul
 ; CHECK-NEXT: fmadd
@@ -86,19 +87,25 @@ define float @food(float %a, double %b) nounwind {
 ; CHECK-NEXT: frsp
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @foo
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+  %x = call fast double @llvm.sqrt.f64(double %b)
+  %y = fptrunc double %x to float
+  %r = fdiv fast float %a, %y
+  ret float %r
 }
 
-define float @goo(float %a, float %b) nounwind {
-  %x = call float @llvm.sqrt.f32(float %b)
-  %r = fdiv float %a, %x
+define float @food_safe(float %a, double %b) nounwind {
+; CHECK: @food_safe
+; CHECK: fsqrt
+; CHECK: fdivs
+; CHECK: blr
+  %x = call double @llvm.sqrt.f64(double %b)
+  %y = fptrunc double %x to float
+  %r = fdiv float %a, %y
   ret float %r
+}
 
-; CHECK: @goo
+define float @goo_fmf(float %a, float %b) nounwind {
+; CHECK: @goo_fmf
 ; CHECK-DAG: frsqrtes
 ; CHECK: fmuls
 ; CHECK-NEXT: fmadds
@@ -106,36 +113,37 @@ define float @goo(float %a, float %b) nounwind {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @goo
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+  %x = call fast float @llvm.sqrt.f32(float %b)
+  %r = fdiv fast float %a, %x
+  ret float %r
 }
 
-
-define float @no_estimate_refinement_f32(float %a, float %b) #0 {
+define float @goo_safe(float %a, float %b) nounwind {
+; CHECK: @goo_safe
+; CHECK: fsqrts
+; CHECK: fdivs
+; CHECK: blr
   %x = call float @llvm.sqrt.f32(float %b)
   %r = fdiv float %a, %x
   ret float %r
+}
 
+define float @no_estimate_refinement_f32(float %a, float %b) #0 {
 ; CHECK-LABEL: @no_estimate_refinement_f32
 ; CHECK: frsqrtes
 ; CHECK-NOT: fmadds
 ; CHECK: fmuls
 ; CHECK-NOT: fmadds
 ; CHECK: blr
+  %x = call fast float @llvm.sqrt.f32(float %b)
+  %r = fdiv fast float %a, %x
+  ret float %r
 }
 
 ; Recognize that this is rsqrt(a) * rcp(b) * c, 
 ; not 1 / ( 1 / sqrt(a)) * rcp(b) * c.
-define float @rsqrt_fmul(float %a, float %b, float %c) {
-  %x = call float @llvm.sqrt.f32(float %a)
-  %y = fmul float %x, %b 
-  %z = fdiv float %c, %y
-  ret float %z
-
-; CHECK: @rsqrt_fmul
+define float @rsqrt_fmul_fmf(float %a, float %b, float %c) {
+; CHECK: @rsqrt_fmul_fmf
 ; CHECK-DAG: frsqrtes
 ; CHECK-DAG: fres
 ; CHECK-DAG: fnmsubs
@@ -146,32 +154,45 @@ define float @rsqrt_fmul(float %a, float %b, float %c) {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
+  %x = call fast float @llvm.sqrt.f32(float %a)
+  %y = fmul fast float %x, %b 
+  %z = fdiv fast float %c, %y
+  ret float %z
+}
+
+; Recognize that this is rsqrt(a) * rcp(b) * c, 
+; not 1 / ( 1 / sqrt(a)) * rcp(b) * c.
+define float @rsqrt_fmul_safe(float %a, float %b, float %c) {
+; CHECK: @rsqrt_fmul_safe
+; CHECK: fsqrts
+; CHECK: fmuls
+; CHECK: fdivs
+; CHECK: blr
+  %x = call float @llvm.sqrt.f32(float %a)
+  %y = fmul float %x, %b 
+  %z = fdiv float %c, %y
+  ret float %z
+}
 
-; CHECK-SAFE: @rsqrt_fmul
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: fmuls
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+define <4 x float> @hoo_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo_fmf
+; CHECK: vrsqrtefp
+  %x = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
+  %r = fdiv fast <4 x float> %a, %x
+  ret <4 x float> %r
 }
 
-define <4 x float> @hoo(<4 x float> %a, <4 x float> %b) nounwind {
+define <4 x float> @hoo_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo_safe
+; CHECK-NOT: vrsqrtefp
+; CHECK: blr
   %x = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %b)
   %r = fdiv <4 x float> %a, %x
   ret <4 x float> %r
-
-; CHECK: @hoo
-; CHECK: vrsqrtefp
-
-; CHECK-SAFE: @hoo
-; CHECK-SAFE-NOT: vrsqrtefp
-; CHECK-SAFE: blr
 }
 
-define double @foo2(double %a, double %b) nounwind {
-  %r = fdiv double %a, %b
-  ret double %r
-
-; CHECK: @foo2
+define double @foo2_fmf(double %a, double %b) nounwind {
+; CHECK: @foo2_fmf
 ; CHECK-DAG: fre
 ; CHECK-DAG: fnmsub
 ; CHECK: fmadd
@@ -179,45 +200,54 @@ define double @foo2(double %a, double %b) nounwind {
 ; CHECK-NEXT: fmadd
 ; CHECK-NEXT: fmul
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @foo2
-; CHECK-SAFE: fdiv
-; CHECK-SAFE: blr
+  %r = fdiv fast double %a, %b
+  ret double %r
 }
 
-define float @goo2(float %a, float %b) nounwind {
-  %r = fdiv float %a, %b
-  ret float %r
+define double @foo2_safe(double %a, double %b) nounwind {
+; CHECK: @foo2_safe
+; CHECK: fdiv
+; CHECK: blr
+  %r = fdiv double %a, %b
+  ret double %r
+}
 
-; CHECK: @goo2
+define float @goo2_fmf(float %a, float %b) nounwind {
+; CHECK: @goo2_fmf
 ; CHECK-DAG: fres
 ; CHECK-DAG: fnmsubs
 ; CHECK: fmadds
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: blr
-
-; CHECK-SAFE: @goo2
-; CHECK-SAFE: fdivs
-; CHECK-SAFE: blr
+  %r = fdiv fast float %a, %b
+  ret float %r
 }
 
-define <4 x float> @hoo2(<4 x float> %a, <4 x float> %b) nounwind {
-  %r = fdiv <4 x float> %a, %b
-  ret <4 x float> %r
+define float @goo2_safe(float %a, float %b) nounwind {
+; CHECK: @goo2_safe
+; CHECK: fdivs
+; CHECK: blr
+  %r = fdiv float %a, %b
+  ret float %r
+}
 
-; CHECK: @hoo2
+define <4 x float> @hoo2_fmf(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo2_fmf
 ; CHECK: vrefp
-
-; CHECK-SAFE: @hoo2
-; CHECK-SAFE-NOT: vrefp
-; CHECK-SAFE: blr
+  %r = fdiv fast <4 x float> %a, %b
+  ret <4 x float> %r
 }
 
-define double @foo3(double %a) nounwind {
-  %r = call double @llvm.sqrt.f64(double %a)
-  ret double %r
+define <4 x float> @hoo2_safe(<4 x float> %a, <4 x float> %b) nounwind {
+; CHECK: @hoo2_safe
+; CHECK-NOT: vrefp
+; CHECK: blr
+  %r = fdiv <4 x float> %a, %b
+  ret <4 x float> %r
+}
 
-; CHECK: @foo3
+define double @foo3_fmf(double %a) nounwind {
+; CHECK: @foo3_fmf
 ; CHECK: fcmpu
 ; CHECK-DAG: frsqrte
 ; CHECK: fmul
@@ -229,17 +259,20 @@ define double @foo3(double %a) nounwind {
 ; CHECK-NEXT: fmul
 ; CHECK-NEXT: fmul
 ; CHECK: blr
-
-; CHECK-SAFE: @foo3
-; CHECK-SAFE: fsqrt
-; CHECK-SAFE: blr
+  %r = call fast double @llvm.sqrt.f64(double %a)
+  ret double %r
 }
 
-define float @goo3(float %a) nounwind {
-  %r = call float @llvm.sqrt.f32(float %a)
-  ret float %r
+define double @foo3_safe(double %a) nounwind {
+; CHECK: @foo3_safe
+; CHECK: fsqrt
+; CHECK: blr
+  %r = call double @llvm.sqrt.f64(double %a)
+  ret double %r
+}
 
-; CHECK: @goo3
+define float @goo3_fmf(float %a) nounwind {
+; CHECK: @goo3_fmf
 ; CHECK: fcmpu
 ; CHECK-DAG: frsqrtes
 ; CHECK: fmuls
@@ -247,24 +280,32 @@ define float @goo3(float %a) nounwind {
 ; CHECK-NEXT: fmuls
 ; CHECK-NEXT: fmuls
 ; CHECK: blr
-
-; CHECK-SAFE: @goo3
-; CHECK-SAFE: fsqrts
-; CHECK-SAFE: blr
+  %r = call fast float @llvm.sqrt.f32(float %a)
+  ret float %r
 }
 
-define <4 x float> @hoo3(<4 x float> %a) nounwind {
-  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
-  ret <4 x float> %r
+define float @goo3_safe(float %a) nounwind {
+; CHECK: @goo3_safe
+; CHECK: fsqrts
+; CHECK: blr
+  %r = call float @llvm.sqrt.f32(float %a)
+  ret float %r
+}
 
-; CHECK: @hoo3
+define <4 x float> @hoo3_fmf(<4 x float> %a) nounwind {
+; CHECK: @hoo3_fmf
 ; CHECK: vrsqrtefp
 ; CHECK-DAG: vcmpeqfp
+  %r = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
+}
 
-; CHECK-SAFE: @hoo3
-; CHECK-SAFE-NOT: vrsqrtefp
-; CHECK-SAFE: blr
+define <4 x float> @hoo3_safe(<4 x float> %a) nounwind {
+; CHECK: @hoo3_safe
+; CHECK-NOT: vrsqrtefp
+; CHECK: blr
+  %r = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a)
+  ret <4 x float> %r
 }
 
 attributes #0 = { nounwind "reciprocal-estimates"="sqrtf:0,sqrtd:0" }
-