diff options
Diffstat (limited to 'llvm/lib')
| -rw-r--r-- | llvm/lib/IR/AutoUpgrade.cpp | 66 | ||||
| -rw-r--r-- | llvm/lib/Target/X86/X86IntrinsicsInfo.h | 30 |
2 files changed, 66 insertions, 30 deletions
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index ce0b10d8a77..52dd90fd9b1 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -194,6 +194,16 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { Name.startswith("x86.sse.storeu.") || Name.startswith("x86.sse2.storeu.") || Name.startswith("x86.avx.storeu.") || + Name.startswith("x86.avx512.mask.storeu.p") || + Name.startswith("x86.avx512.mask.storeu.b.") || + Name.startswith("x86.avx512.mask.storeu.w.") || + Name.startswith("x86.avx512.mask.storeu.d.") || + Name.startswith("x86.avx512.mask.storeu.q.") || + Name.startswith("x86.avx512.mask.store.p") || + Name.startswith("x86.avx512.mask.store.b.") || + Name.startswith("x86.avx512.mask.store.w.") || + Name.startswith("x86.avx512.mask.store.d.") || + Name.startswith("x86.avx512.mask.store.q.") || Name == "x86.sse42.crc32.64.8" || Name.startswith("x86.avx.vbroadcast.s") || Name.startswith("x86.sse2.psll.dq") || @@ -358,6 +368,40 @@ static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, LLVMContext &C, return Builder.CreateBitCast(Res, ResultTy, "cast"); } +static Value *UpgradeMaskedStore(IRBuilder<> &Builder, LLVMContext &C, + Value *Ptr, Value *Data, Value *Mask, + bool Aligned) { + // Cast the pointer to the right type. + Ptr = Builder.CreateBitCast(Ptr, + llvm::PointerType::getUnqual(Data->getType())); + unsigned Align = + Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1; + + // If the mask is all ones just emit a regular store. + if (const auto *C = dyn_cast<Constant>(Mask)) + if (C->isAllOnesValue()) + return Builder.CreateAlignedStore(Data, Ptr, Align); + + // Convert the mask from an integer type to a vector of i1. + unsigned NumElts = Data->getType()->getVectorNumElements(); + llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(), + cast<IntegerType>(Mask->getType())->getBitWidth()); + Mask = Builder.CreateBitCast(Mask, MaskTy); + + // If we have less than 8 elements, then the starting mask was an i8 and + // we need to extract down to the right number of elements. + if (NumElts < 8) { + int Indices[4]; + for (unsigned i = 0; i != NumElts; ++i) + Indices[i] = i; + Mask = Builder.CreateShuffleVector(Mask, Mask, + makeArrayRef(Indices, NumElts), + "extract"); + } + + return Builder.CreateMaskedStore(Data, Ptr, Align, Mask); +} + // UpgradeIntrinsicCall - Upgrade a call to an old intrinsic to be a call the // upgraded intrinsic. All argument and return casting must be provided in // order to seamlessly integrate with existing context. @@ -459,6 +503,28 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { // Remove intrinsic. CI->eraseFromParent(); return; + } else if (Name.startswith("llvm.x86.avx512.mask.storeu.p") || + Name.startswith("llvm.x86.avx512.mask.storeu.b.") || + Name.startswith("llvm.x86.avx512.mask.storeu.w.") || + Name.startswith("llvm.x86.avx512.mask.storeu.d.") || + Name.startswith("llvm.x86.avx512.mask.storeu.q.")) { + UpgradeMaskedStore(Builder, C, CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2), /*Aligned*/false); + + // Remove intrinsic. + CI->eraseFromParent(); + return; + } else if (Name.startswith("llvm.x86.avx512.mask.store.p") || + Name.startswith("llvm.x86.avx512.mask.store.b.") || + Name.startswith("llvm.x86.avx512.mask.store.w.") || + Name.startswith("llvm.x86.avx512.mask.store.d.") || + Name.startswith("llvm.x86.avx512.mask.store.q.")) { + UpgradeMaskedStore(Builder, C, CI->getArgOperand(0), CI->getArgOperand(1), + CI->getArgOperand(2), /*Aligned*/true); + + // Remove intrinsic. + CI->eraseFromParent(); + return; } else if (Name.startswith("llvm.x86.xop.vpcom")) { Intrinsic::ID intID; if (Name.endswith("ub")) diff --git a/llvm/lib/Target/X86/X86IntrinsicsInfo.h b/llvm/lib/Target/X86/X86IntrinsicsInfo.h index 6d8f254b5c4..c4c5c1c95ab 100644 --- a/llvm/lib/Target/X86/X86IntrinsicsInfo.h +++ b/llvm/lib/Target/X86/X86IntrinsicsInfo.h @@ -214,36 +214,6 @@ static const IntrinsicData IntrinsicsWithChain[] = { X86ISD::VTRUNC, 0), X86_INTRINSIC_DATA(avx512_mask_pmov_wb_mem_512, TRUNCATE_TO_MEM_VI8, X86ISD::VTRUNC, 0), - X86_INTRINSIC_DATA(avx512_mask_store_d_128, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_d_256, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_d_512, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_pd_128, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_pd_256, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_pd_512, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_ps_128, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_ps_256, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_ps_512, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_q_128, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_q_256, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_store_q_512, STOREA, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_b_128, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_b_256, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_b_512, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_d_128, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_d_256, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_d_512, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_pd_128, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_pd_256, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_pd_512, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_ps_128, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_ps_256, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_ps_512, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_q_128, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_q_256, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_q_512, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_w_128, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_w_256, STOREU, ISD::DELETED_NODE, 0), - X86_INTRINSIC_DATA(avx512_mask_storeu_w_512, STOREU, ISD::DELETED_NODE, 0), X86_INTRINSIC_DATA(avx512_scatter_dpd_512, SCATTER, X86::VSCATTERDPDZmr, 0), X86_INTRINSIC_DATA(avx512_scatter_dpi_512, SCATTER, X86::VPSCATTERDDZmr, 0), X86_INTRINSIC_DATA(avx512_scatter_dpq_512, SCATTER, X86::VPSCATTERDQZmr, 0), |
