diff options
Diffstat (limited to 'llvm/lib')
22 files changed, 167 insertions, 54 deletions
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index d8be4ad42ad..e3157603203 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -648,6 +648,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(nonnull); KEYWORD(noredzone); KEYWORD(noreturn); + KEYWORD(nocf_check); KEYWORD(nounwind); KEYWORD(optnone); KEYWORD(optsize); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 29e6b07a76b..1c607de4427 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -1131,6 +1131,7 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B, case lltok::kw_nonlazybind: B.addAttribute(Attribute::NonLazyBind); break; case lltok::kw_noredzone: B.addAttribute(Attribute::NoRedZone); break; case lltok::kw_noreturn: B.addAttribute(Attribute::NoReturn); break; + case lltok::kw_nocf_check: B.addAttribute(Attribute::NoCfCheck); break; case lltok::kw_norecurse: B.addAttribute(Attribute::NoRecurse); break; case lltok::kw_nounwind: B.addAttribute(Attribute::NoUnwind); break; case lltok::kw_optnone: B.addAttribute(Attribute::OptimizeNone); break; @@ -1468,6 +1469,7 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) { case lltok::kw_nonlazybind: case lltok::kw_noredzone: case lltok::kw_noreturn: + case lltok::kw_nocf_check: case lltok::kw_nounwind: case lltok::kw_optnone: case lltok::kw_optsize: @@ -1561,6 +1563,7 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) { case lltok::kw_nonlazybind: case lltok::kw_noredzone: case lltok::kw_noreturn: + case lltok::kw_nocf_check: case lltok::kw_nounwind: case lltok::kw_optnone: case lltok::kw_optsize: diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h index ad826cc4fd2..3ef44d48ec9 100644 --- a/llvm/lib/AsmParser/LLToken.h +++ b/llvm/lib/AsmParser/LLToken.h @@ -199,6 +199,7 @@ enum Kind { kw_nonnull, kw_noredzone, kw_noreturn, + kw_nocf_check, kw_nounwind, kw_optnone, kw_optsize, diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 582298136ae..74b57a86b8b 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1160,6 +1160,7 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { case Attribute::Speculatable: return 1ULL << 54; case Attribute::StrictFP: return 1ULL << 55; case Attribute::SanitizeHWAddress: return 1ULL << 56; + case Attribute::NoCfCheck: return 1ULL << 57; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -1338,6 +1339,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoRedZone; case bitc::ATTR_KIND_NO_RETURN: return Attribute::NoReturn; + case bitc::ATTR_KIND_NOCF_CHECK: + return Attribute::NoCfCheck; case bitc::ATTR_KIND_NO_UNWIND: return Attribute::NoUnwind; case bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE: diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index 0738b907be3..d82101b1413 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -638,6 +638,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NO_RED_ZONE; case Attribute::NoReturn: return bitc::ATTR_KIND_NO_RETURN; + case Attribute::NoCfCheck: + return bitc::ATTR_KIND_NOCF_CHECK; case Attribute::NoUnwind: return bitc::ATTR_KIND_NO_UNWIND; case Attribute::OptimizeForSize: diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index cf3e9297e3c..7a50b0940ae 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -3706,7 +3706,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, TLI.getPICJumpTableRelocBase(Table, DAG)); } - Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); + + Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); Results.push_back(Tmp1); break; } diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp index 30216bcde68..0aea24f22c0 100644 --- a/llvm/lib/IR/Attributes.cpp +++ b/llvm/lib/IR/Attributes.cpp @@ -299,6 +299,8 @@ std::string Attribute::getAsString(bool InAttrGrp) const { return "noredzone"; if (hasAttribute(Attribute::NoReturn)) return "noreturn"; + if (hasAttribute(Attribute::NoCfCheck)) + return "nocf_check"; if (hasAttribute(Attribute::NoRecurse)) return "norecurse"; if (hasAttribute(Attribute::NoUnwind)) diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index b9df7c91a64..aec5c682296 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -1404,6 +1404,7 @@ Verifier::visitModuleFlag(const MDNode *Op, static bool isFuncOnlyAttr(Attribute::AttrKind Kind) { switch (Kind) { case Attribute::NoReturn: + case Attribute::NoCfCheck: case Attribute::NoUnwind: case Attribute::NoInline: case Attribute::AlwaysInline: diff --git a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp index 2e45d9eee12..dd48602d358 100644 --- a/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp +++ b/llvm/lib/Target/X86/AsmParser/X86AsmParser.cpp @@ -2353,21 +2353,22 @@ bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name, .Cases("acquire", "release", isParsingIntelSyntax()) .Default(false); - auto isLockRepeatPrefix = [](StringRef N) { + auto isLockRepeatNtPrefix = [](StringRef N) { return StringSwitch<bool>(N) - .Cases("lock", "rep", "repe", "repz", "repne", "repnz", true) + .Cases("lock", "rep", "repe", "repz", "repne", "repnz", "notrack", true) .Default(false); }; bool CurlyAsEndOfStatement = false; unsigned Flags = X86::IP_NO_PREFIX; - while (isLockRepeatPrefix(Name.lower())) { + while (isLockRepeatNtPrefix(Name.lower())) { unsigned Prefix = StringSwitch<unsigned>(Name) .Cases("lock", "lock", X86::IP_HAS_LOCK) .Cases("rep", "repe", "repz", X86::IP_HAS_REPEAT) .Cases("repne", "repnz", X86::IP_HAS_REPEAT_NE) + .Cases("notrack", "notrack", X86::IP_HAS_NOTRACK) .Default(X86::IP_NO_PREFIX); // Invalid prefix (impossible) Flags |= Prefix; if (getLexer().is(AsmToken::EndOfStatement)) { diff --git a/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp b/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp index 5860285378e..5941e06d76c 100644 --- a/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp +++ b/llvm/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp @@ -54,6 +54,9 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, raw_ostream &OS, if ((TSFlags & X86II::LOCK) || (Flags & X86::IP_HAS_LOCK)) OS << "\tlock\t"; + if ((TSFlags & X86II::NOTRACK) || (Flags & X86::IP_HAS_NOTRACK)) + OS << "\tnotrack\t"; + if (Flags & X86::IP_HAS_REPEAT_NE) OS << "\trepne\t"; else if (Flags & X86::IP_HAS_REPEAT) diff --git a/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp b/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp index ccfe14098f0..0e0a02b150a 100644 --- a/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp +++ b/llvm/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp @@ -49,6 +49,9 @@ void X86IntelInstPrinter::printInst(const MCInst *MI, raw_ostream &OS, else if (Flags & X86::IP_HAS_REPEAT) OS << "\trep\t"; + if ((TSFlags & X86II::NOTRACK) || (Flags & X86::IP_HAS_NOTRACK)) + OS << "\tnotrack\t"; + printInstruction(MI, OS); // Next always print the annotation. diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h index 07cc488d047..95d7200d13c 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h +++ b/llvm/lib/Target/X86/MCTargetDesc/X86BaseInfo.h @@ -60,8 +60,9 @@ namespace X86 { IP_HAS_REPEAT_NE = 4, IP_HAS_REPEAT = 8, IP_HAS_LOCK = 16, - NO_SCHED_INFO = 32 // Don't add sched comment to the current instr because - // it was already added + NO_SCHED_INFO = 32, // Don't add sched comment to the current instr because + // it was already added + IP_HAS_NOTRACK = 64 }; } // end namespace X86; @@ -572,7 +573,11 @@ namespace X86II { /// Explicitly specified rounding control EVEX_RCShift = Has3DNow0F0FOpcodeShift + 1, - EVEX_RC = 1ULL << EVEX_RCShift + EVEX_RC = 1ULL << EVEX_RCShift, + + // NOTRACK prefix + NoTrackShift = EVEX_RCShift + 1, + NOTRACK = 1ULL << NoTrackShift }; // getBaseOpcodeFor - This function returns the "base" X86 opcode for the diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp index 608dc4acb04..3ba8b500dfd 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp @@ -1134,6 +1134,10 @@ bool X86MCCodeEmitter::emitOpcodePrefix(uint64_t TSFlags, unsigned &CurByte, if (TSFlags & X86II::LOCK || MI.getFlags() & X86::IP_HAS_LOCK) EmitByte(0xF0, CurByte, OS); + // Emit the NOTRACK opcode prefix. + if (TSFlags & X86II::NOTRACK || MI.getFlags() & X86::IP_HAS_NOTRACK) + EmitByte(0x3E, CurByte, OS); + switch (TSFlags & X86II::OpPrefixMask) { case X86II::PD: // 66 EmitByte(0x66, CurByte, OS); diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp index 943022585a1..3806d2dad3d 100644 --- a/llvm/lib/Target/X86/X86FastISel.cpp +++ b/llvm/lib/Target/X86/X86FastISel.cpp @@ -3174,6 +3174,13 @@ bool X86FastISel::fastLowerCall(CallLoweringInfo &CLI) { CLI.CS ? dyn_cast<CallInst>(CLI.CS->getInstruction()) : nullptr; const Function *CalledFn = CI ? CI->getCalledFunction() : nullptr; + // Call / invoke instructions with NoCfCheck attribute require special + // handling. + const auto *II = + CLI.CS ? dyn_cast<InvokeInst>(CLI.CS->getInstruction()) : nullptr; + if ((CI && CI->doesNoCfCheck()) || (II && II->doesNoCfCheck())) + return false; + // Functions with no_caller_saved_registers that need special handling. if ((CI && CI->hasFnAttr("no_caller_saved_registers")) || (CalledFn && CalledFn->hasFnAttribute("no_caller_saved_registers"))) diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 0df09187d0e..f0981687a64 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3415,6 +3415,11 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, const Function *Fn = CI ? CI->getCalledFunction() : nullptr; bool HasNCSR = (CI && CI->hasFnAttr("no_caller_saved_registers")) || (Fn && Fn->hasFnAttribute("no_caller_saved_registers")); + const auto *II = dyn_cast_or_null<InvokeInst>(CLI.CS.getInstruction()); + bool HasNoCfCheck = + (CI && CI->doesNoCfCheck()) || (II && II->doesNoCfCheck()); + const Module *M = MF.getMMI().getModule(); + Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); if (CallConv == CallingConv::X86_INTR) report_fatal_error("X86 interrupts may not be called directly"); @@ -3898,7 +3903,11 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, return DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops); } - Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); + if (HasNoCfCheck && IsCFProtectionSupported) { + Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops); + } else { + Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); + } InFlag = Chain.getValue(1); // Create the CALLSEQ_END node. @@ -25852,6 +25861,8 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const { case X86ISD::GF2P8MULB: return "X86ISD::GF2P8MULB"; case X86ISD::GF2P8AFFINEQB: return "X86ISD::GF2P8AFFINEQB"; case X86ISD::GF2P8AFFINEINVQB: return "X86ISD::GF2P8AFFINEINVQB"; + case X86ISD::NT_CALL: return "X86ISD::NT_CALL"; + case X86ISD::NT_BRIND: return "X86ISD::NT_BRIND"; } return nullptr; } @@ -38709,6 +38720,22 @@ void X86TargetLowering::finalizeLowering(MachineFunction &MF) const { TargetLoweringBase::finalizeLowering(MF); } +SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc& dl, + SDValue Value, SDValue Addr, + SelectionDAG &DAG) const { + const Module *M = DAG.getMachineFunction().getMMI().getModule(); + Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch"); + if (IsCFProtectionSupported) { + // In case control-flow branch protection is enabled, we need to add + // notrack prefix to the indirect branch. + // In order to do that we create NT_BRIND SDNode. + // Upon ISEL, the pattern will convert it to jmp with NoTrack prefix. + return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Value, Addr); + } + + return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, DAG); +} + /// This method query the target whether it is beneficial for dag combiner to /// promote the specified node. If true, it should return the desired promotion /// type by reference. diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 6181dbd2a6a..2fdc0c22e39 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -75,6 +75,9 @@ namespace llvm { /// CALL, + /// Same as call except it adds the NoTrack prefix. + NT_CALL, + /// This operation implements the lowering for readcyclecounter. RDTSC_DAG, @@ -122,6 +125,10 @@ namespace llvm { /// or TEST instruction. BRCOND, + /// BRIND node with NoTrack prefix. Operand 0 is the chain operand and + /// operand 1 is the target address. + NT_BRIND, + /// Return with a flag operand. Operand 0 is the chain operand, operand /// 1 is the number of bytes of stack to pop. RET_FLAG, @@ -1114,9 +1121,12 @@ namespace llvm { bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override; - void finalizeLowering(MachineFunction &MF) const override; + SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value, + SDValue Addr, SelectionDAG &DAG) + const override; + protected: std::pair<const TargetRegisterClass *, uint8_t> findRepresentativeClass(const TargetRegisterInfo *TRI, diff --git a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp index f280d3db6fe..ab401fa0258 100644 --- a/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp +++ b/llvm/lib/Target/X86/X86IndirectBranchTracking.cpp @@ -22,7 +22,6 @@ #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" using namespace llvm; @@ -58,7 +57,8 @@ private: /// Adds a new ENDBR instruction to the begining of the MBB. /// The function will not add it if already exists. /// It will add ENDBR32 or ENDBR64 opcode, depending on the target. - void addENDBR(MachineBasicBlock &MBB) const; + /// \returns true if the ENDBR was added and false otherwise. + bool addENDBR(MachineBasicBlock &MBB) const; }; } // end anonymous namespace @@ -69,7 +69,7 @@ FunctionPass *llvm::createX86IndirectBranchTrackingPass() { return new X86IndirectBranchTrackingPass(); } -void X86IndirectBranchTrackingPass::addENDBR(MachineBasicBlock &MBB) const { +bool X86IndirectBranchTrackingPass::addENDBR(MachineBasicBlock &MBB) const { assert(TII && "Target instruction info was not initialized"); assert((X86::ENDBR64 == EndbrOpcode || X86::ENDBR32 == EndbrOpcode) && "Unexpected Endbr opcode"); @@ -80,13 +80,16 @@ void X86IndirectBranchTrackingPass::addENDBR(MachineBasicBlock &MBB) const { if (MI == MBB.end() || EndbrOpcode != MI->getOpcode()) { BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(EndbrOpcode)); NumEndBranchAdded++; + return true; } + + return false; } bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) { const X86Subtarget &SubTarget = MF.getSubtarget<X86Subtarget>(); - // Make sure that the target supports ENDBR instruction. + // Make sure that the target supports IBT instruction. if (!SubTarget.hasIBT()) return false; @@ -103,50 +106,20 @@ bool X86IndirectBranchTrackingPass::runOnMachineFunction(MachineFunction &MF) { EndbrOpcode = SubTarget.is64Bit() ? X86::ENDBR64 : X86::ENDBR32; // Non-internal function or function whose address was taken, can be - // invoked through indirect calls. Mark the first BB with ENDBR instruction. - // TODO: Do not add ENDBR instruction in case notrack attribute is used. - if (MF.getFunction().hasAddressTaken() || - !MF.getFunction().hasLocalLinkage()) { + // accessed through indirect calls. Mark the first BB with ENDBR instruction + // unless nocf_check attribute is used. + if ((MF.getFunction().hasAddressTaken() || + !MF.getFunction().hasLocalLinkage()) && + !MF.getFunction().doesNoCfCheck()) { auto MBB = MF.begin(); - addENDBR(*MBB); - Changed = true; + Changed |= addENDBR(*MBB); } - for (auto &MBB : MF) { - // Find all basic blocks that thier address was taken (for example + for (auto &MBB : MF) + // Find all basic blocks that their address was taken (for example // in the case of indirect jump) and add ENDBR instruction. - if (MBB.hasAddressTaken()) { - addENDBR(MBB); - Changed = true; - } - } - - // Adds ENDBR instructions to MBB destinations of the jump table. - // TODO: In case of more than 50 destinations, do not add ENDBR and - // instead add DS_PREFIX. - if (MachineJumpTableInfo *JTI = MF.getJumpTableInfo()) { - for (const auto &JT : JTI->getJumpTables()) { - for (auto *MBB : JT.MBBs) { - // This assert verifies the assumption that this MBB has an indirect - // jump terminator in one of its predecessor. - // Jump tables are generated when lowering switch-case statements or - // setjmp/longjump functions. As a result only indirect jumps use jump - // tables. - #ifndef NDEBUG - bool hasIndirectJumpTerm = false; - for (auto &PredMBB : MBB->predecessors()) - for (auto &TermI : PredMBB->terminators()) - if (TermI.isIndirectBranch()) - hasIndirectJumpTerm = true; - assert(hasIndirectJumpTerm && - "The MBB is not the destination of an indirect jump"); - (void)hasIndirectJumpTerm; - #endif - addENDBR(*MBB); - Changed = true; - } - } - } + if (MBB.hasAddressTaken()) + Changed |= addENDBR(MBB); return Changed; } diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td index de3b3709104..9fba65c6cf6 100644 --- a/llvm/lib/Target/X86/X86InstrControl.td +++ b/llvm/lib/Target/X86/X86InstrControl.td @@ -153,6 +153,30 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in { [(brind (loadi64 addr:$dst))], IIC_JMP_MEM>, Requires<[In64BitMode]>, Sched<[WriteJumpLd]>; + let isCodeGenOnly = 1, Predicates = [HasIBT] in { + def JMP16r_NT : I<0xFF, MRM4r, (outs), (ins GR16 : $dst), "jmp{w}\t{*}$dst", + [(X86NoTrackBrind GR16 : $dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>, + OpSize16, Sched<[WriteJump]>, NOTRACK; + + def JMP16m_NT : I<0xFF, MRM4m, (outs), (ins i16mem : $dst), "jmp{w}\t{*}$dst", + [(X86NoTrackBrind (loadi16 addr : $dst))], IIC_JMP_MEM>, + Requires<[Not64BitMode]>, OpSize16, Sched<[WriteJumpLd]>, NOTRACK; + + def JMP32r_NT : I<0xFF, MRM4r, (outs), (ins GR32 : $dst), "jmp{l}\t{*}$dst", + [(X86NoTrackBrind GR32 : $dst)], IIC_JMP_REG>, Requires<[Not64BitMode]>, + OpSize32, Sched<[WriteJump]>, NOTRACK; + def JMP32m_NT : I<0xFF, MRM4m, (outs), (ins i32mem : $dst), "jmp{l}\t{*}$dst", + [(X86NoTrackBrind (loadi32 addr : $dst))], IIC_JMP_MEM>, + Requires<[Not64BitMode]>, OpSize32, Sched<[WriteJumpLd]>, NOTRACK; + + def JMP64r_NT : I<0xFF, MRM4r, (outs), (ins GR64 : $dst), "jmp{q}\t{*}$dst", + [(X86NoTrackBrind GR64 : $dst)], IIC_JMP_REG>, Requires<[In64BitMode]>, + Sched<[WriteJump]>, NOTRACK; + def JMP64m_NT : I<0xFF, MRM4m, (outs), (ins i64mem : $dst), "jmp{q}\t{*}$dst", + [(X86NoTrackBrind(loadi64 addr : $dst))], IIC_JMP_MEM>, + Requires<[In64BitMode]>, Sched<[WriteJumpLd]>, NOTRACK; + } + let Predicates = [Not64BitMode] in { def FARJMP16i : Iseg16<0xEA, RawFrmImm16, (outs), (ins i16imm:$off, i16imm:$seg), @@ -175,7 +199,6 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in { Sched<[WriteJumpLd]>; } - // Loop instructions let SchedRW = [WriteJump] in { def LOOP : Ii8PCRel<0xE2, RawFrm, (outs), (ins brtarget8:$dst), "loop\t$dst", [], IIC_LOOP>; @@ -219,6 +242,25 @@ let isCall = 1 in Requires<[Not64BitMode,FavorMemIndirectCall,NotUseRetpoline]>, Sched<[WriteJumpLd]>; + let isCodeGenOnly = 1, Predicates = [HasIBT] in { + def CALL16r_NT : I<0xFF, MRM2r, (outs), (ins GR16 : $dst), + "call{w}\t{*}$dst",[(X86NoTrackCall GR16 : $dst)], IIC_CALL_RI>, + OpSize16, Requires<[Not64BitMode]>, Sched<[WriteJump]>, NOTRACK; + def CALL16m_NT : I<0xFF, MRM2m, (outs), (ins i16mem : $dst), + "call{w}\t{*}$dst",[(X86NoTrackCall(loadi16 addr : $dst))], + IIC_CALL_MEM>, OpSize16, + Requires<[Not64BitMode,FavorMemIndirectCall]>, + Sched<[WriteJumpLd]>, NOTRACK; + def CALL32r_NT : I<0xFF, MRM2r, (outs), (ins GR32 : $dst), + "call{l}\t{*}$dst",[(X86NoTrackCall GR32 : $dst)], IIC_CALL_RI>, + OpSize32, Requires<[Not64BitMode]>, Sched<[WriteJump]>, NOTRACK; + def CALL32m_NT : I<0xFF, MRM2m, (outs), (ins i32mem : $dst), + "call{l}\t{*}$dst",[(X86NoTrackCall(loadi32 addr : $dst))], + IIC_CALL_MEM>, OpSize32, + Requires<[Not64BitMode,FavorMemIndirectCall]>, + Sched<[WriteJumpLd]>, NOTRACK; + } + let Predicates = [Not64BitMode] in { def FARCALL16i : Iseg16<0x9A, RawFrmImm16, (outs), (ins i16imm:$off, i16imm:$seg), @@ -306,6 +348,17 @@ let isCall = 1, Uses = [RSP, SSP], SchedRW = [WriteJump] in { Requires<[In64BitMode,FavorMemIndirectCall, NotUseRetpoline]>; + let isCodeGenOnly = 1, Predicates = [HasIBT] in{ + def CALL64r_NT : I<0xFF, MRM2r, (outs), (ins GR64 : $dst), + "call{q}\t{*}$dst",[(X86NoTrackCall GR64 : $dst)], + IIC_CALL_RI>, + Requires<[In64BitMode]>, NOTRACK; + def CALL64m_NT : I<0xFF, MRM2m, (outs), (ins i64mem : $dst), + "call{q}\t{*}$dst",[(X86NoTrackCall(loadi64 addr : $dst))], + IIC_CALL_MEM>, + Requires<[In64BitMode,FavorMemIndirectCall]>, NOTRACK; + } + def FARCALL64 : RI<0xFF, MRM3m, (outs), (ins opaque80mem:$dst), "lcall{q}\t{*}$dst", [], IIC_CALL_FAR_MEM>; } diff --git a/llvm/lib/Target/X86/X86InstrFormats.td b/llvm/lib/Target/X86/X86InstrFormats.td index 0b266e5591b..53f71127d2b 100644 --- a/llvm/lib/Target/X86/X86InstrFormats.td +++ b/llvm/lib/Target/X86/X86InstrFormats.td @@ -215,6 +215,7 @@ class EVEX_RC { bit hasEVEX_RC = 1; } class EVEX_V512 { bit hasEVEX_L2 = 1; bit hasVEX_L = 0; } class EVEX_V256 { bit hasEVEX_L2 = 0; bit hasVEX_L = 1; } class EVEX_V128 { bit hasEVEX_L2 = 0; bit hasVEX_L = 0; } +class NOTRACK { bit hasNoTrackPrefix = 1; } // Specify AVX512 8-bit compressed displacement encoding based on the vector // element size in bits (8, 16, 32, 64) and the CDisp8 form. @@ -296,6 +297,7 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, int CD8_EltSize = 0; // Compressed disp8 form - element-size in bytes. bit has3DNow0F0FOpcode =0;// Wacky 3dNow! encoding? bit hasEVEX_RC = 0; // Explicitly specified rounding control in FP instruction. + bit hasNoTrackPrefix = 0; // Does this inst has 0x3E (NoTrack) prefix? bits<2> EVEX_LL; let EVEX_LL{0} = hasVEX_L; @@ -347,6 +349,7 @@ class X86Inst<bits<8> opcod, Format f, ImmType i, dag outs, dag ins, let TSFlags{52-46} = CD8_Scale; let TSFlags{53} = has3DNow0F0FOpcode; let TSFlags{54} = hasEVEX_RC; + let TSFlags{55} = hasNoTrackPrefix; } class PseudoI<dag oops, dag iops, list<dag> pattern, diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td index 82062dd0b4f..72709fd1d94 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.td +++ b/llvm/lib/Target/X86/X86InstrInfo.td @@ -94,6 +94,8 @@ def SDT_X86CallSeqEnd : SDCallSeqEnd<[SDTCisVT<0, i32>, def SDT_X86Call : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; +def SDT_X86NtBrind : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; + def SDT_X86VASTART_SAVE_XMM_REGS : SDTypeProfile<0, -1, [SDTCisVT<0, i8>, SDTCisVT<1, iPTR>, SDTCisVT<2, iPTR>]>; @@ -196,6 +198,12 @@ def X86call : SDNode<"X86ISD::CALL", SDT_X86Call, [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, SDNPVariadic]>; +def X86NoTrackCall : SDNode<"X86ISD::NT_CALL", SDT_X86Call, + [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue, + SDNPVariadic]>; +def X86NoTrackBrind : SDNode<"X86ISD::NT_BRIND", SDT_X86NtBrind, + [SDNPHasChain]>; + def X86rep_stos: SDNode<"X86ISD::REP_STOS", SDTX86RepStr, [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore]>; def X86rep_movs: SDNode<"X86ISD::REP_MOVS", SDTX86RepStr, diff --git a/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp b/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp index 325a5d77aad..d10b4f4aaeb 100644 --- a/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp +++ b/llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp @@ -42,6 +42,7 @@ static Attribute::AttrKind parseAttrKind(StringRef Kind) { .Case("nonlazybind", Attribute::NonLazyBind) .Case("noredzone", Attribute::NoRedZone) .Case("noreturn", Attribute::NoReturn) + .Case("nocf_check", Attribute::NoCfCheck) .Case("norecurse", Attribute::NoRecurse) .Case("nounwind", Attribute::NoUnwind) .Case("optnone", Attribute::OptimizeNone) diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index 2fc987d860d..ef39f21632c 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -696,6 +696,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs, case Attribute::StackProtectStrong: case Attribute::StrictFP: case Attribute::UWTable: + case Attribute::NoCfCheck: break; } |
