[X86] Use BEXTR/BEXTRI for 64-bit 'and' with a large mask

Summary: The 64-bit 'and' with immediate instruction only supports a 32-bit immediate. So for larger constants we have to load the constant into a register first. If the immediate happens to be a mask we can use the BEXTRI instruction to perform the masking. We already do something similar using the BZHI instruction from the BMI2 instruction set.

Reviewers: RKSimon, spatel

Reviewed By: RKSimon

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D36129

llvm-svn: 309706

3 files changed, 60 insertions, 9 deletions

diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index 75638f24bfe..1fc174750c7 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -844,6 +844,7 @@ def HasFMA       : Predicate<"Subtarget->hasFMA()">;
 def HasFMA4      : Predicate<"Subtarget->hasFMA4()">;
 def HasXOP       : Predicate<"Subtarget->hasXOP()">;
 def HasTBM       : Predicate<"Subtarget->hasTBM()">;
+def NoTBM        : Predicate<"!Subtarget->hasTBM()">;
 def HasLWP       : Predicate<"Subtarget->hasLWP()">;
 def HasMOVBE     : Predicate<"Subtarget->hasMOVBE()">;
 def HasRDRAND    : Predicate<"Subtarget->hasRDRAND()">;
@@ -853,6 +854,7 @@ def HasFSGSBase  : Predicate<"Subtarget->hasFSGSBase()">;
 def HasLZCNT     : Predicate<"Subtarget->hasLZCNT()">;
 def HasBMI       : Predicate<"Subtarget->hasBMI()">;
 def HasBMI2      : Predicate<"Subtarget->hasBMI2()">;
+def NoBMI2       : Predicate<"!Subtarget->hasBMI2()">;
 def HasVBMI      : Predicate<"Subtarget->hasVBMI()">,
                      AssemblerPredicate<"FeatureVBMI", "AVX-512 VBMI ISA">;
 def HasIFMA      : Predicate<"Subtarget->hasIFMA()">,
@@ -2352,20 +2354,40 @@ def CountTrailingOnes : SDNodeXForm<imm, [{
   return getI8Imm(countTrailingOnes(N->getZExtValue()), SDLoc(N));
 }]>;
 
-def BZHIMask : ImmLeaf<i64, [{
-  return isMask_64(Imm) && (countTrailingOnes<uint64_t>(Imm) > 32);
+def BEXTRMaskXForm : SDNodeXForm<imm, [{
+  unsigned Length = countTrailingOnes(N->getZExtValue());
+  return getI32Imm(Length << 8, SDLoc(N));
 }]>;
 
-let Predicates = [HasBMI2] in {
-  def : Pat<(and GR64:$src, BZHIMask:$mask),
+def AndMask64 : ImmLeaf<i64, [{
+  return isMask_64(Imm) && Imm > UINT32_MAX;
+}]>;
+
+// Use BEXTR for 64-bit 'and' with large immediate 'mask'.
+let Predicates = [HasBMI, NoBMI2, NoTBM] in {
+  def : Pat<(and GR64:$src, AndMask64:$mask),
+            (BEXTR64rr GR64:$src,
+              (SUBREG_TO_REG (i64 0),
+                             (MOV32ri (BEXTRMaskXForm imm:$mask)), sub_32bit))>;
+  def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
+            (BEXTR64rm addr:$src,
+              (SUBREG_TO_REG (i64 0),
+                             (MOV32ri (BEXTRMaskXForm imm:$mask)), sub_32bit))>;
+}
+
+// Use BZHI for 64-bit 'and' with large immediate 'mask'.
+let Predicates = [HasBMI2, NoTBM] in {
+  def : Pat<(and GR64:$src, AndMask64:$mask),
             (BZHI64rr GR64:$src,
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                              (MOV8ri (CountTrailingOnes imm:$mask)), sub_8bit))>;
-  def : Pat<(and (loadi64 addr:$src), BZHIMask:$mask),
+  def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
             (BZHI64rm addr:$src,
               (INSERT_SUBREG (i64 (IMPLICIT_DEF)),
                              (MOV8ri (CountTrailingOnes imm:$mask)), sub_8bit))>;
+}
 
+let Predicates = [HasBMI2] in {
   def : Pat<(and GR32:$src, (add (shl 1, GR8:$lz), -1)),
             (BZHI32rr GR32:$src,
               (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$lz, sub_8bit))>;
@@ -2511,6 +2533,15 @@ defm T1MSKC  : tbm_binary_intr<0x01, "t1mskc", MRM7r, MRM7m>;
 defm TZMSK   : tbm_binary_intr<0x01, "tzmsk", MRM4r, MRM4m>;
 } // HasTBM, EFLAGS
 
+// Use BEXTRI for 64-bit 'and' with large immediate 'mask'.
+let Predicates = [HasTBM] in {
+  def : Pat<(and GR64:$src, AndMask64:$mask),
+            (BEXTRI64ri GR64:$src, (BEXTRMaskXForm imm:$mask))>;
+
+  def : Pat<(and (loadi64 addr:$src), AndMask64:$mask),
+            (BEXTRI64mi addr:$src, (BEXTRMaskXForm imm:$mask))>;
+}
+
 //===----------------------------------------------------------------------===//
 // Lightweight Profiling Instructions
 
diff --git a/llvm/test/CodeGen/X86/bmi.ll b/llvm/test/CodeGen/X86/bmi.ll
index a7dbd8bdf99..50e9ab5f145 100644
--- a/llvm/test/CodeGen/X86/bmi.ll
+++ b/llvm/test/CodeGen/X86/bmi.ll
@@ -612,8 +612,8 @@ entry:
 define i64 @bzhi64_constant_mask(i64 %x) {
 ; BMI1-LABEL: bzhi64_constant_mask:
 ; BMI1:       # BB#0: # %entry
-; BMI1-NEXT:    movabsq $4611686018427387903, %rax # imm = 0x3FFFFFFFFFFFFFFF
-; BMI1-NEXT:    andq %rdi, %rax
+; BMI1-NEXT:    movl $15872, %eax # imm = 0x3E00
+; BMI1-NEXT:    bextrq %rax, %rdi, %rax
 ; BMI1-NEXT:    retq
 ;
 ; BMI2-LABEL: bzhi64_constant_mask:
@@ -629,8 +629,8 @@ entry:
 define i64 @bzhi64_constant_mask_load(i64* %x) {
 ; BMI1-LABEL: bzhi64_constant_mask_load:
 ; BMI1:       # BB#0: # %entry
-; BMI1-NEXT:    movabsq $4611686018427387903, %rax # imm = 0x3FFFFFFFFFFFFFFF
-; BMI1-NEXT:    andq (%rdi), %rax
+; BMI1-NEXT:    movl $15872, %eax # imm = 0x3E00
+; BMI1-NEXT:    bextrq %rax, (%rdi), %rax
 ; BMI1-NEXT:    retq
 ;
 ; BMI2-LABEL: bzhi64_constant_mask_load:
diff --git a/llvm/test/CodeGen/X86/tbm_patterns.ll b/llvm/test/CodeGen/X86/tbm_patterns.ll
index 5ce6bbd4b49..f110bd538bf 100644
--- a/llvm/test/CodeGen/X86/tbm_patterns.ll
+++ b/llvm/test/CodeGen/X86/tbm_patterns.ll
@@ -253,3 +253,23 @@ define i64 @test_x86_tbm_tzmsk_u64(i64 %a) nounwind {
   ret i64 %t2
 }
 
+define i64 @test_and_large_constant_mask(i64 %x) {
+; CHECK-LABEL: test_and_large_constant_mask:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    bextr $15872, %rdi, %rax # imm = 0x3E00
+; CHECK-NEXT:    retq
+entry:
+  %and = and i64 %x, 4611686018427387903
+  ret i64 %and
+}
+
+define i64 @test_and_large_constant_mask_load(i64* %x) {
+; CHECK-LABEL: test_and_large_constant_mask_load:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    bextr $15872, (%rdi), %rax # imm = 0x3E00
+; CHECK-NEXT:    retq
+entry:
+  %x1 = load i64, i64* %x
+  %and = and i64 %x1, 4611686018427387903
+  ret i64 %and
+}