summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Target/ARM/ARMAsmPrinter.cpp33
-rw-r--r--llvm/lib/Target/ARM/ARMAsmPrinter.h13
-rw-r--r--llvm/lib/Target/ARM/ARMConstantIslandPass.cpp6
-rw-r--r--llvm/lib/Target/ARM/ARMConstantPoolValue.cpp12
-rw-r--r--llvm/lib/Target/ARM/ARMConstantPoolValue.h20
-rw-r--r--llvm/lib/Target/ARM/ARMISelLowering.cpp170
-rw-r--r--llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp2
-rw-r--r--llvm/lib/Target/ARM/ARMMachineFunctionInfo.h25
8 files changed, 275 insertions, 6 deletions
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index db6eefdb999..27d2c4b356d 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -97,6 +97,13 @@ void ARMAsmPrinter::EmitXXStructor(const DataLayout &DL, const Constant *CV) {
OutStreamer->EmitValue(E, Size);
}
+void ARMAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
+ if (PromotedGlobals.count(GV))
+ // The global was promoted into a constant pool. It should not be emitted.
+ return;
+ AsmPrinter::EmitGlobalVariable(GV);
+}
+
/// runOnMachineFunction - This uses the EmitInstruction()
/// method to print assembly for each instruction.
///
@@ -109,6 +116,12 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
const Function* F = MF.getFunction();
const TargetMachine& TM = MF.getTarget();
+ // Collect all globals that had their storage promoted to a constant pool.
+ // Functions are emitted before variables, so this accumulates promoted
+ // globals from all functions in PromotedGlobals.
+ for (auto *GV : AFI->getGlobalsPromotedToConstantPool())
+ PromotedGlobals.insert(GV);
+
// Calculate this function's optimization goal.
unsigned OptimizationGoal;
if (F->hasFnAttribute(Attribute::OptimizeNone))
@@ -1002,6 +1015,26 @@ EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) {
ARMConstantPoolValue *ACPV = static_cast<ARMConstantPoolValue*>(MCPV);
+ if (ACPV->isPromotedGlobal()) {
+ // This constant pool entry is actually a global whose storage has been
+ // promoted into the constant pool. This global may be referenced still
+ // by debug information, and due to the way AsmPrinter is set up, the debug
+ // info is immutable by the time we decide to promote globals to constant
+ // pools. Because of this, we need to ensure we emit a symbol for the global
+ // with private linkage (the default) so debug info can refer to it.
+ //
+ // However, if this global is promoted into several functions we must ensure
+ // we don't try and emit duplicate symbols!
+ auto *ACPC = cast<ARMConstantPoolConstant>(ACPV);
+ auto *GV = ACPC->getPromotedGlobal();
+ if (!EmittedPromotedGlobalLabels.count(GV)) {
+ MCSymbol *GVSym = getSymbol(GV);
+ OutStreamer->EmitLabel(GVSym);
+ EmittedPromotedGlobalLabels.insert(GV);
+ }
+ return EmitGlobalConstant(DL, ACPC->getPromotedGlobalInit());
+ }
+
MCSymbol *MCSym;
if (ACPV->isLSDA()) {
MCSym = getCurExceptionSym();
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.h b/llvm/lib/Target/ARM/ARMAsmPrinter.h
index fe4a2bfca35..9eafde55d4d 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.h
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.h
@@ -56,6 +56,16 @@ class LLVM_LIBRARY_VISIBILITY ARMAsmPrinter : public AsmPrinter {
/// -1 if uninitialized, 0 if conflicting goals
int OptimizationGoals;
+ /// List of globals that have had their storage promoted to a constant
+ /// pool. This lives between calls to runOnMachineFunction and collects
+ /// data from every MachineFunction. It is used during doFinalization
+ /// when all non-function globals are emitted.
+ SmallPtrSet<const GlobalVariable*,2> PromotedGlobals;
+ /// Set of globals in PromotedGlobals that we've emitted labels for.
+ /// We need to emit labels even for promoted globals so that DWARF
+ /// debug info can link properly.
+ SmallPtrSet<const GlobalVariable*,2> EmittedPromotedGlobalLabels;
+
public:
explicit ARMAsmPrinter(TargetMachine &TM,
std::unique_ptr<MCStreamer> Streamer);
@@ -90,7 +100,8 @@ public:
void EmitStartOfAsmFile(Module &M) override;
void EmitEndOfAsmFile(Module &M) override;
void EmitXXStructor(const DataLayout &DL, const Constant *CV) override;
-
+ void EmitGlobalVariable(const GlobalVariable *GV) override;
+
// lowerOperand - Convert a MachineOperand into the equivalent MCOperand.
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp);
diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index fd679f0f6b0..d2e0725581d 100644
--- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -767,6 +767,7 @@ initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
case ARM::LDRi12:
case ARM::LDRcp:
case ARM::t2LDRpci:
+ case ARM::t2LDRHpci:
Bits = 12; // +-offset_12
NegOk = true;
break;
@@ -782,6 +783,11 @@ initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs) {
Scale = 4; // +-(offset_8*4)
NegOk = true;
break;
+
+ case ARM::tLDRHi:
+ Bits = 5;
+ Scale = 2; // +(offset_5*2)
+ break;
}
// Remember that this is a user of a CP entry.
diff --git a/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp b/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp
index f13ae481bdb..05d1b84b328 100644
--- a/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantPoolValue.cpp
@@ -131,6 +131,12 @@ ARMConstantPoolConstant::ARMConstantPoolConstant(const Constant *C,
AddCurrentAddress),
CVal(C) {}
+ARMConstantPoolConstant::ARMConstantPoolConstant(const GlobalVariable *GV,
+ const Constant *C)
+ : ARMConstantPoolValue((Type *)C->getType(), 0, ARMCP::CPPromotedGlobal, 0,
+ ARMCP::no_modifier, false),
+ CVal(C), GVar(GV) {}
+
ARMConstantPoolConstant *
ARMConstantPoolConstant::Create(const Constant *C, unsigned ID) {
return new ARMConstantPoolConstant(C, ID, ARMCP::CPValue, 0,
@@ -138,6 +144,12 @@ ARMConstantPoolConstant::Create(const Constant *C, unsigned ID) {
}
ARMConstantPoolConstant *
+ARMConstantPoolConstant::Create(const GlobalVariable *GVar,
+ const Constant *Initializer) {
+ return new ARMConstantPoolConstant(GVar, Initializer);
+}
+
+ARMConstantPoolConstant *
ARMConstantPoolConstant::Create(const GlobalValue *GV,
ARMCP::ARMCPModifier Modifier) {
return new ARMConstantPoolConstant((Type*)Type::getInt32Ty(GV->getContext()),
diff --git a/llvm/lib/Target/ARM/ARMConstantPoolValue.h b/llvm/lib/Target/ARM/ARMConstantPoolValue.h
index fae64dc2455..581bf8b1d74 100644
--- a/llvm/lib/Target/ARM/ARMConstantPoolValue.h
+++ b/llvm/lib/Target/ARM/ARMConstantPoolValue.h
@@ -24,6 +24,7 @@ namespace llvm {
class BlockAddress;
class Constant;
class GlobalValue;
+class GlobalVariable;
class LLVMContext;
class MachineBasicBlock;
@@ -33,7 +34,8 @@ namespace ARMCP {
CPExtSymbol,
CPBlockAddress,
CPLSDA,
- CPMachineBasicBlock
+ CPMachineBasicBlock,
+ CPPromotedGlobal
};
enum ARMCPModifier {
@@ -103,7 +105,8 @@ public:
bool isBlockAddress() const { return Kind == ARMCP::CPBlockAddress; }
bool isLSDA() const { return Kind == ARMCP::CPLSDA; }
bool isMachineBasicBlock() const{ return Kind == ARMCP::CPMachineBasicBlock; }
-
+ bool isPromotedGlobal() const{ return Kind == ARMCP::CPPromotedGlobal; }
+
int getExistingMachineCPValue(MachineConstantPool *CP,
unsigned Alignment) override;
@@ -133,6 +136,7 @@ inline raw_ostream &operator<<(raw_ostream &O, const ARMConstantPoolValue &V) {
/// Functions, and BlockAddresses.
class ARMConstantPoolConstant : public ARMConstantPoolValue {
const Constant *CVal; // Constant being loaded.
+ const GlobalVariable *GVar = nullptr;
ARMConstantPoolConstant(const Constant *C,
unsigned ID,
@@ -146,11 +150,14 @@ class ARMConstantPoolConstant : public ARMConstantPoolValue {
unsigned char PCAdj,
ARMCP::ARMCPModifier Modifier,
bool AddCurrentAddress);
+ ARMConstantPoolConstant(const GlobalVariable *GV, const Constant *Init);
public:
static ARMConstantPoolConstant *Create(const Constant *C, unsigned ID);
static ARMConstantPoolConstant *Create(const GlobalValue *GV,
ARMCP::ARMCPModifier Modifier);
+ static ARMConstantPoolConstant *Create(const GlobalVariable *GV,
+ const Constant *Initializer);
static ARMConstantPoolConstant *Create(const Constant *C, unsigned ID,
ARMCP::ARMCPKind Kind,
unsigned char PCAdj);
@@ -162,6 +169,12 @@ public:
const GlobalValue *getGV() const;
const BlockAddress *getBlockAddress() const;
+ const GlobalVariable *getPromotedGlobal() const {
+ return dyn_cast_or_null<GlobalVariable>(GVar);
+ }
+ const Constant *getPromotedGlobalInit() const {
+ return CVal;
+ }
int getExistingMachineCPValue(MachineConstantPool *CP,
unsigned Alignment) override;
@@ -174,7 +187,8 @@ public:
void print(raw_ostream &O) const override;
static bool classof(const ARMConstantPoolValue *APV) {
- return APV->isGlobalValue() || APV->isBlockAddress() || APV->isLSDA();
+ return APV->isGlobalValue() || APV->isBlockAddress() || APV->isLSDA() ||
+ APV->isPromotedGlobal();
}
bool equals(const ARMConstantPoolConstant *A) const {
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 434b4546483..018fb1617d6 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -37,6 +37,7 @@
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
+#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
@@ -59,12 +60,28 @@ using namespace llvm;
STATISTIC(NumTailCalls, "Number of tail calls");
STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
+STATISTIC(NumConstpoolPromoted,
+ "Number of constants with their storage promoted into constant pools");
static cl::opt<bool>
ARMInterworking("arm-interworking", cl::Hidden,
cl::desc("Enable / disable ARM interworking (for debugging only)"),
cl::init(true));
+static cl::opt<bool> EnableConstpoolPromotion(
+ "arm-promote-constant", cl::Hidden,
+ cl::desc("Enable / disable promotion of unnamed_addr constants into "
+ "constant pools"),
+ cl::init(true));
+static cl::opt<unsigned> ConstpoolPromotionMaxSize(
+ "arm-promote-constant-max-size", cl::Hidden,
+ cl::desc("Maximum size of constant to promote into a constant pool"),
+ cl::init(64));
+static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
+ "arm-promote-constant-max-total", cl::Hidden,
+ cl::desc("Maximum size of ALL constants to promote into a constant pool"),
+ cl::init(128));
+
namespace {
class ARMCCState : public CCState {
public:
@@ -2963,6 +2980,154 @@ ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
llvm_unreachable("bogus TLS model");
}
+/// Return true if all users of V are within function F, looking through
+/// ConstantExprs.
+static bool allUsersAreInFunction(const Value *V, const Function *F) {
+ SmallVector<const User*,4> Worklist;
+ for (auto *U : V->users())
+ Worklist.push_back(U);
+ while (!Worklist.empty()) {
+ auto *U = Worklist.pop_back_val();
+ if (isa<ConstantExpr>(U)) {
+ for (auto *UU : U->users())
+ Worklist.push_back(UU);
+ continue;
+ }
+
+ auto *I = dyn_cast<Instruction>(U);
+ if (!I || I->getParent()->getParent() != F)
+ return false;
+ }
+ return true;
+}
+
+/// Return true if all users of V are within some (any) function, looking through
+/// ConstantExprs. In other words, are there any global constant users?
+static bool allUsersAreInFunctions(const Value *V) {
+ SmallVector<const User*,4> Worklist;
+ for (auto *U : V->users())
+ Worklist.push_back(U);
+ while (!Worklist.empty()) {
+ auto *U = Worklist.pop_back_val();
+ if (isa<ConstantExpr>(U)) {
+ for (auto *UU : U->users())
+ Worklist.push_back(UU);
+ continue;
+ }
+
+ if (!isa<Instruction>(U))
+ return false;
+ }
+ return true;
+}
+
+// Return true if T is an integer, float or an array/vector of either.
+static bool isSimpleType(Type *T) {
+ if (T->isIntegerTy() || T->isFloatingPointTy())
+ return true;
+ Type *SubT = nullptr;
+ if (T->isArrayTy())
+ SubT = T->getArrayElementType();
+ else if (T->isVectorTy())
+ SubT = T->getVectorElementType();
+ else
+ return false;
+ return SubT->isIntegerTy() || SubT->isFloatingPointTy();
+}
+
+static SDValue promoteToConstantPool(const GlobalValue *GV, SelectionDAG &DAG,
+ EVT PtrVT, SDLoc dl) {
+ // If we're creating a pool entry for a constant global with unnamed address,
+ // and the global is small enough, we can emit it inline into the constant pool
+ // to save ourselves an indirection.
+ //
+ // This is a win if the constant is only used in one function (so it doesn't
+ // need to be duplicated) or duplicating the constant wouldn't increase code
+ // size (implying the constant is no larger than 4 bytes).
+ const Function *F = DAG.getMachineFunction().getFunction();
+
+ // We rely on this decision to inline being idemopotent and unrelated to the
+ // use-site. We know that if we inline a variable at one use site, we'll
+ // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
+ // doesn't know about this optimization, so bail out if it's enabled else
+ // we could decide to inline here (and thus never emit the GV) but require
+ // the GV from fast-isel generated code.
+ if (!EnableConstpoolPromotion ||
+ DAG.getMachineFunction().getTarget().Options.EnableFastISel)
+ return SDValue();
+
+ auto *GVar = dyn_cast<GlobalVariable>(GV);
+ if (!GVar || !GVar->hasInitializer() ||
+ !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
+ !GVar->hasLocalLinkage())
+ return SDValue();
+
+ // Ensure that we don't try and inline any type that contains pointers. If
+ // we inline a value that contains relocations, we move the relocations from
+ // .data to .text which is not ideal.
+ auto *Init = GVar->getInitializer();
+ if (!isSimpleType(Init->getType()))
+ return SDValue();
+
+ // The constant islands pass can only really deal with alignment requests
+ // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
+ // any type wanting greater alignment requirements than 4 bytes. We also
+ // can only promote constants that are multiples of 4 bytes in size or
+ // are paddable to a multiple of 4. Currently we only try and pad constants
+ // that are strings for simplicity.
+ auto *CDAInit = dyn_cast<ConstantDataArray>(Init);
+ unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
+ unsigned Align = DAG.getDataLayout().getABITypeAlignment(Init->getType());
+ unsigned RequiredPadding = 4 - (Size % 4);
+ bool PaddingPossible =
+ RequiredPadding == 4 || (CDAInit && CDAInit->isString());
+ if (!PaddingPossible || Align > 4 || Size > ConstpoolPromotionMaxSize)
+ return SDValue();
+
+ unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
+ MachineFunction &MF = DAG.getMachineFunction();
+ ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+
+ // We can't bloat the constant pool too much, else the ConstantIslands pass
+ // may fail to converge. If we haven't promoted this global yet (it may have
+ // multiple uses), and promoting it would increase the constant pool size (Sz
+ // > 4), ensure we have space to do so up to MaxTotal.
+ if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4)
+ if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
+ ConstpoolPromotionMaxTotal)
+ return SDValue();
+
+ // This is only valid if all users are in a single function OR it has users
+ // in multiple functions but it no larger than a pointer. We also check if
+ // GVar has constant (non-ConstantExpr) users. If so, it essentially has its
+ // address taken.
+ if (!allUsersAreInFunction(GVar, F) &&
+ !(Size <= 4 && allUsersAreInFunctions(GVar)))
+ return SDValue();
+
+ // We're going to inline this global. Pad it out if needed.
+ if (RequiredPadding != 4) {
+ StringRef S = CDAInit->getAsString();
+
+ SmallVector<uint8_t,16> V(S.size());
+ std::copy(S.bytes_begin(), S.bytes_end(), V.begin());
+ while (RequiredPadding--)
+ V.push_back(0);
+ Init = ConstantDataArray::get(*DAG.getContext(), V);
+ }
+
+ auto CPVal = ARMConstantPoolConstant::Create(GVar, Init);
+ SDValue CPAddr =
+ DAG.getTargetConstantPool(CPVal, PtrVT, /*Align=*/4);
+ if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) {
+ AFI->markGlobalAsPromotedToConstantPool(GVar);
+ AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() +
+ PaddedSize - 4);
+ }
+ ++NumConstpoolPromoted;
+ return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
+}
+
SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
SelectionDAG &DAG) const {
EVT PtrVT = getPointerTy(DAG.getDataLayout());
@@ -2974,6 +3139,11 @@ SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
bool IsRO =
(isa<GlobalVariable>(GV) && cast<GlobalVariable>(GV)->isConstant()) ||
isa<Function>(GV);
+
+ if (TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
+ if (SDValue V = promoteToConstantPool(GV, DAG, PtrVT, dl))
+ return V;
+
if (isPositionIndependent()) {
bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV);
diff --git a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp
index b6dee9ff838..72e37a38425 100644
--- a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp
@@ -21,4 +21,4 @@ ARMFunctionInfo::ARMFunctionInfo(MachineFunction &MF)
FramePtrSpillOffset(0), GPRCS1Offset(0), GPRCS2Offset(0), DPRCSOffset(0),
GPRCS1Size(0), GPRCS2Size(0), DPRCSSize(0),
PICLabelUId(0), VarArgsFrameIndex(0), HasITBlocks(false),
- ArgumentStackSize(0), IsSplitCSR(false) {}
+ ArgumentStackSize(0), IsSplitCSR(false), PromotedGlobalsIncrease(0) {}
diff --git a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
index f71497240ff..8c485e89bf5 100644
--- a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
+++ b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
@@ -121,6 +121,12 @@ class ARMFunctionInfo : public MachineFunctionInfo {
/// copies.
bool IsSplitCSR;
+ /// Globals that have had their storage promoted into the constant pool.
+ SmallPtrSet<const GlobalVariable*,2> PromotedGlobals;
+
+ /// The amount the literal pool has been increasedby due to promoted globals.
+ int PromotedGlobalsIncrease;
+
public:
ARMFunctionInfo() :
isThumb(false),
@@ -131,7 +137,8 @@ public:
FramePtrSpillOffset(0), GPRCS1Offset(0), GPRCS2Offset(0), DPRCSOffset(0),
GPRCS1Size(0), GPRCS2Size(0), DPRCSAlignGapSize(0), DPRCSSize(0),
NumAlignedDPRCS2Regs(0), PICLabelUId(0),
- VarArgsFrameIndex(0), HasITBlocks(false), IsSplitCSR(false) {}
+ VarArgsFrameIndex(0), HasITBlocks(false), IsSplitCSR(false),
+ PromotedGlobalsIncrease(0) {}
explicit ARMFunctionInfo(MachineFunction &MF);
@@ -226,6 +233,22 @@ public:
}
return It;
}
+
+ /// Indicate to the backend that \c GV has had its storage changed to inside
+ /// a constant pool. This means it no longer needs to be emitted as a
+ /// global variable.
+ void markGlobalAsPromotedToConstantPool(const GlobalVariable *GV) {
+ PromotedGlobals.insert(GV);
+ }
+ SmallPtrSet<const GlobalVariable*, 2>& getGlobalsPromotedToConstantPool() {
+ return PromotedGlobals;
+ }
+ int getPromotedConstpoolIncrease() const {
+ return PromotedGlobalsIncrease;
+ }
+ void setPromotedConstpoolIncrease(int Sz) {
+ PromotedGlobalsIncrease = Sz;
+ }
};
} // End llvm namespace
OpenPOWER on IntegriCloud