[NVPTX] Allow libcalls that are defined in the current module.

The patch adds a possibility to make library calls on NVPTX.

An important thing about library functions - they must be defined within
the current module. This basically should guarantee that we produce a
valid PTX assembly (without calls to not defined functions). The one who
wants to use the libcalls is probably will have to link against
compiler-rt or any other implementation.

Currently, it's completely impossible to make library calls because of
error LLVM ERROR: Cannot select: i32 = ExternalSymbol '...'. But we can
lower ExternalSymbol to TargetExternalSymbol and verify if the function
definition is available.

Also, there was an issue with a DAG during legalisation. When we expand
instruction into libcall, the inner call-chain isn't being "integrated"
into outer chain. Since the last "data-flow" (call retval load) node is
located in call-chain earlier than CALLSEQ_END node, the latter becomes
a leaf and therefore a dead node (and is being removed quite fast).
Proposed here solution relies on another data-flow pseudo nodes
(ProxyReg) which purpose is only to keep CALLSEQ_END at legalisation and
instruction selection phases - we remove the pseudo instructions before
register scheduling phase.

Patch by Denys Zariaiev!

Differential Revision: https://reviews.llvm.org/D34708

llvm-svn: 350069

7 files changed, 273 insertions, 12 deletions

diff --git a/llvm/test/CodeGen/NVPTX/calls-with-phi.ll b/llvm/test/CodeGen/NVPTX/calls-with-phi.ll
new file mode 100644
index 00000000000..6e010ea9adc
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/calls-with-phi.ll
@@ -0,0 +1,22 @@
+; RUN: llc < %s -march=nvptx 2>&1 | FileCheck %s
+; Make sure the example doesn't crash with segfault
+
+; CHECK: .visible .func ({{.*}}) loop
+define i32 @loop(i32, i32) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %0, %entry ], [ %res, %loop ]
+  %res = call i32 @div(i32 %i, i32 %1)
+
+  %exitcond = icmp eq i32 %res, %0
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret i32 %res
+}
+
+define i32 @div(i32, i32) {
+  ret i32 0
+}
diff --git a/llvm/test/CodeGen/NVPTX/libcall-fulfilled.ll b/llvm/test/CodeGen/NVPTX/libcall-fulfilled.ll
new file mode 100644
index 00000000000..9d6777d59c3
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/libcall-fulfilled.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s -march=nvptx 2>&1 | FileCheck %s
+; Allow to make libcalls that are defined in the current module
+
+; Underlying libcall declaration
+; CHECK: .visible .func  (.param .align 16 .b8 func_retval0[16]) __umodti3
+
+define i128 @remainder(i128, i128) {
+bb0:
+  ; CHECK:      { // callseq 0, 0
+  ; CHECK:      call.uni (retval0),
+  ; CHECK-NEXT: __umodti3,
+  ; CHECK-NEXT: (
+  ; CHECK-NEXT: param0,
+  ; CHECK-NEXT: param1
+  ; CHECK-NEXT: );
+  ; CHECK-NEXT: ld.param.v2.b64 {%[[REG0:rd[0-9]+]], %[[REG1:rd[0-9]+]]}, [retval0+0];
+  ; CHECK-NEXT: } // callseq 0
+  %a = urem i128 %0, %1
+  br label %bb1
+
+bb1:
+  ; CHECK-NEXT: st.param.v2.b64 [func_retval0+0], {%[[REG0]], %[[REG1]]};
+  ; CHECK-NEXT: ret;
+  ret i128 %a
+}
+
+; Underlying libcall definition
+; CHECK: .visible .func  (.param .align 16 .b8 func_retval0[16]) __umodti3(
+define i128 @__umodti3(i128, i128) {
+  ret i128 0
+}
diff --git a/llvm/test/CodeGen/NVPTX/libcall-instruction.ll b/llvm/test/CodeGen/NVPTX/libcall-instruction.ll
index 0c2cab7eaa5..a40a504c94c 100644
--- a/llvm/test/CodeGen/NVPTX/libcall-instruction.ll
+++ b/llvm/test/CodeGen/NVPTX/libcall-instruction.ll
@@ -1,7 +1,7 @@
 ; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s
-; used to panic on failed assetion and now fails with a "Cannot select"
+; used to panic on failed assertion and now fails with an "Undefined external symbol"
 
-; CHECK: LLVM ERROR: Cannot select: {{t28|0x[0-9a-f]+}}: i32 = ExternalSymbol'__umodti3'
+; CHECK: LLVM ERROR: Undefined external symbol "__umodti3"
 define hidden i128 @remainder(i128, i128) {
   %3 = urem i128 %0, %1
   ret i128 %3
diff --git a/llvm/test/CodeGen/NVPTX/libcall-intrinsic.ll b/llvm/test/CodeGen/NVPTX/libcall-intrinsic.ll
new file mode 100644
index 00000000000..0b5e0224399
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/libcall-intrinsic.ll
@@ -0,0 +1,10 @@
+; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s
+; used to seqfault and now fails with an "Undefined external symbol"
+
+; CHECK: LLVM ERROR: Undefined external symbol "__powidf2"
+define double @powi(double, i32) {
+  %a = call double @llvm.powi.f64(double %0, i32 %1)
+  ret double %a
+}
+
+declare double @llvm.powi.f64(double, i32) nounwind readnone
diff --git a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-mir.ll b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-mir.ll
new file mode 100644
index 00000000000..6bfbe2aea81
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-mir.ll
@@ -0,0 +1,25 @@
+; RUN: llc -march=nvptx64 -stop-before=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=MIR --check-prefix=MIR-BEFORE
+
+; RUN: llc -march=nvptx64 -stop-after=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=MIR --check-prefix=MIR-AFTER
+
+; Check ProxyRegErasure pass MIR manipulation.
+
+declare <4 x i32> @callee_vec_i32()
+define  <4 x i32> @check_vec_i32() {
+  ; MIR: body:
+  ; MIR-DAG: Callseq_Start {{[0-9]+}}, {{[0-9]+}}
+  ; MIR-DAG: %0:int32regs, %1:int32regs, %2:int32regs, %3:int32regs = LoadParamMemV4I32 0
+  ; MIR-DAG: Callseq_End {{[0-9]+}}
+
+  ; MIR-BEFORE-DAG: %4:int32regs = ProxyRegI32 killed %0
+  ; MIR-BEFORE-DAG: %5:int32regs = ProxyRegI32 killed %1
+  ; MIR-BEFORE-DAG: %6:int32regs = ProxyRegI32 killed %2
+  ; MIR-BEFORE-DAG: %7:int32regs = ProxyRegI32 killed %3
+  ; MIR-BEFORE-DAG: StoreRetvalV4I32 killed %4, killed %5, killed %6, killed %7, 0
+  ; MIR-AFTER-DAG:  StoreRetvalV4I32 killed %0, killed %1, killed %2, killed %3, 0
+
+  %ret = call <4 x i32> @callee_vec_i32()
+  ret <4 x i32> %ret
+}
diff --git a/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll
new file mode 100644
index 00000000000..7a3b29b20c4
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll
@@ -0,0 +1,183 @@
+; RUN: llc -march=nvptx64 -stop-before=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | llc -x mir -march=nvptx64 -start-before=nvptx-proxyreg-erasure 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=PTX --check-prefix=PTX-WITH
+
+; RUN: llc -march=nvptx64 -stop-before=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | llc -x mir -march=nvptx64 -start-after=nvptx-proxyreg-erasure 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=PTX --check-prefix=PTX-WITHOUT
+
+; Thorough testing of ProxyRegErasure: PTX assembly with and without the pass.
+
+declare i1 @callee_i1()
+define i1 @check_i1() {
+  ; PTX-LABEL: check_i1
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: and.b32 [[RES:%r[0-9]+]], [[PROXY]], 1;
+  ; PTX-WITH-DAG:    and.b32 [[RES:%r[0-9]+]], [[LD]], 1;
+
+  ; PTX-DAG: st.param.b32 [func_retval0+0], [[RES]];
+
+  %ret = call i1 @callee_i1()
+  ret i1 %ret
+}
+
+declare i16 @callee_i16()
+define  i16 @check_i16() {
+  ; PTX-LABEL: check_i16
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: and.b32 [[RES:%r[0-9]+]], [[PROXY]], 65535;
+  ; PTX-WITH-DAG:    and.b32 [[RES:%r[0-9]+]], [[LD]], 65535;
+
+  ; PTX-DAG: st.param.b32 [func_retval0+0], [[RES]];
+
+  %ret = call i16 @callee_i16()
+  ret i16 %ret
+}
+
+declare i32 @callee_i32()
+define  i32 @check_i32() {
+  ; PTX-LABEL: check_i32
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b32 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b32 [func_retval0+0], [[LD]];
+
+  %ret = call i32 @callee_i32()
+  ret i32 %ret
+}
+
+declare i64 @callee_i64()
+define  i64 @check_i64() {
+  ; PTX-LABEL: check_i64
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b64 [[LD:%rd[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b64 [[PROXY:%rd[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b64 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b64 [func_retval0+0], [[LD]];
+
+  %ret = call i64 @callee_i64()
+  ret i64 %ret
+}
+
+declare i128 @callee_i128()
+define  i128 @check_i128() {
+  ; PTX-LABEL: check_i128
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.v2.b64 {[[LD0:%rd[0-9]+]], [[LD1:%rd[0-9]+]]}, [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b64 [[PROXY0:%rd[0-9]+]], [[LD0]];
+  ; PTX-WITHOUT-DAG: mov.b64 [[PROXY1:%rd[0-9]+]], [[LD1]];
+  ; PTX-WITHOUT-DAG: st.param.v2.b64 [func_retval0+0], {[[PROXY0]], [[PROXY1]]};
+  ; PTX-WITH-DAG:    st.param.v2.b64 [func_retval0+0], {[[LD0]], [[LD1]]};
+
+  %ret = call i128 @callee_i128()
+  ret i128 %ret
+}
+
+declare half @callee_f16()
+define  half @check_f16() {
+  ; PTX-LABEL: check_f16
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b16 [[LD:%h[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b16 [[PROXY:%h[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b16 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b16 [func_retval0+0], [[LD]];
+
+  %ret = call half @callee_f16()
+  ret half %ret
+}
+
+declare float @callee_f32()
+define  float @check_f32() {
+  ; PTX-LABEL: check_f32
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.f32 [[LD:%f[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.f32 [[PROXY:%f[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.f32 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.f32 [func_retval0+0], [[LD]];
+
+  %ret = call float @callee_f32()
+  ret float %ret
+}
+
+declare double @callee_f64()
+define  double @check_f64() {
+  ; PTX-LABEL: check_f64
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.f64 [[LD:%fd[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.f64 [[PROXY:%fd[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.f64 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.f64 [func_retval0+0], [[LD]];
+
+  %ret = call double @callee_f64()
+  ret double %ret
+}
+
+declare <4 x i32> @callee_vec_i32()
+define  <4 x i32> @check_vec_i32() {
+  ; PTX-LABEL: check_vec_i32
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.v4.b32 {[[LD0:%r[0-9]+]], [[LD1:%r[0-9]+]], [[LD2:%r[0-9]+]], [[LD3:%r[0-9]+]]}, [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY0:%r[0-9]+]], [[LD0]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY1:%r[0-9]+]], [[LD1]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY2:%r[0-9]+]], [[LD2]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY3:%r[0-9]+]], [[LD3]];
+  ; PTX-WITHOUT-DAG: st.param.v4.b32 [func_retval0+0], {[[PROXY0]], [[PROXY1]], [[PROXY2]], [[PROXY3]]};
+  ; PTX-WITH-DAG:    st.param.v4.b32 [func_retval0+0], {[[LD0]], [[LD1]], [[LD2]], [[LD3]]};
+
+  %ret = call <4 x i32> @callee_vec_i32()
+  ret <4 x i32> %ret
+}
+
+declare <2 x half> @callee_vec_f16()
+define  <2 x half> @check_vec_f16() {
+  ; PTX-LABEL: check_vec_f16
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%hh[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%hh[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b32 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b32 [func_retval0+0], [[LD]];
+
+  %ret = call <2 x half> @callee_vec_f16()
+  ret <2 x half> %ret
+}
+
+declare <2 x double> @callee_vec_f64()
+define  <2 x double> @check_vec_f64() {
+  ; PTX-LABEL: check_vec_f64
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.v2.f64 {[[LD0:%fd[0-9]+]], [[LD1:%fd[0-9]+]]}, [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.f64 [[PROXY0:%fd[0-9]+]], [[LD0]];
+  ; PTX-WITHOUT-DAG: mov.f64 [[PROXY1:%fd[0-9]+]], [[LD1]];
+  ; PTX-WITHOUT-DAG: st.param.v2.f64 [func_retval0+0], {[[PROXY0]], [[PROXY1]]};
+  ; PTX-WITH-DAG:    st.param.v2.f64 [func_retval0+0], {[[LD0]], [[LD1]]};
+
+  %ret = call <2 x double> @callee_vec_f64()
+  ret <2 x double> %ret
+}
diff --git a/llvm/test/CodeGen/NVPTX/zero-cs.ll b/llvm/test/CodeGen/NVPTX/zero-cs.ll
deleted file mode 100644
index 7a7a99030d4..00000000000
--- a/llvm/test/CodeGen/NVPTX/zero-cs.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s
-; used to seqfault and now fails with a "Cannot select"
-
-; CHECK: LLVM ERROR: Cannot select: {{t7|0x[0-9a-f]+}}: i32 = ExternalSymbol'__powidf2'
-define double @powi() {
-  %1 = call double @llvm.powi.f64(double 1.000000e+00, i32 undef)
-  ret double %1
-}
-
-declare double @llvm.powi.f64(double, i32) nounwind readnone