17 files changed, 289 insertions, 15 deletions
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 33cf6adc4ee..4dd7157d04e 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -4589,6 +4589,25 @@ Examples:
     !2 = !{ i8 0, i8 2, i8 3, i8 6 }
     !3 = !{ i8 -2, i8 0, i8 3, i8 6 }
 
+'``absolute_symbol``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``absolute_symbol`` metadata may be attached to a global variable
+declaration. It marks the declaration as a reference to an absolute symbol,
+which causes the backend to use absolute relocations for the symbol even
+in position independent code, and expresses the possible ranges that the
+global variable's *address* (not its value) is in, in the same format as
+``range`` metadata.
+
+Example:
+
+.. code-block:: llvm
+
+      @a = external global i8, !absolute_symbol !0 ; Absolute symbol in range [0,256)
+
+    ...
+    !0 = !{ i64 0, i64 256 }
+
 '``unpredictable``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/llvm/include/llvm/IR/GlobalValue.h b/llvm/include/llvm/IR/GlobalValue.h
index b973389827c..93970075614 100644
--- a/llvm/include/llvm/IR/GlobalValue.h
+++ b/llvm/include/llvm/IR/GlobalValue.h
@@ -33,6 +33,7 @@
 namespace llvm {
 
 class Comdat;
+class ConstantRange;
 class Error;
 class GlobalObject;
 class Module;
@@ -511,6 +512,13 @@ public:
   }
   GlobalObject *getBaseObject();
 
+  /// Returns whether this is a reference to an absolute symbol.
+  bool isAbsoluteSymbolRef() const;
+
+  /// If this is an absolute symbol reference, returns the range of the symbol,
+  /// otherwise returns None.
+  Optional<ConstantRange> getAbsoluteSymbolRange() const;
+
   /// This method unlinks 'this' from the containing module, but does not delete
   /// it.
   virtual void removeFromParent() = 0;
diff --git a/llvm/include/llvm/IR/LLVMContext.h b/llvm/include/llvm/IR/LLVMContext.h
index ca5ff9227e0..7f43d5df3c3 100644
--- a/llvm/include/llvm/IR/LLVMContext.h
+++ b/llvm/include/llvm/IR/LLVMContext.h
@@ -77,6 +77,7 @@ public:
     MD_loop = 18,                     // "llvm.loop"
     MD_type = 19,                     // "type"
     MD_section_prefix = 20,           // "section_prefix"
+    MD_absolute_symbol = 21,          // "absolute_symbol"
   };
 
   /// Known operand bundle tag IDs, which always have the same value.  All
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index e7220f875e5..950a2fbcff9 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -3345,11 +3345,11 @@ bool llvm::isKnownNonNull(const Value *V) {
   if (const Argument *A = dyn_cast<Argument>(V))
     return A->hasByValOrInAllocaAttr() || A->hasNonNullAttr();
 
-  // A global variable in address space 0 is non null unless extern weak.
-  // Other address spaces may have null as a valid address for a global,
-  // so we can't assume anything.
+  // A global variable in address space 0 is non null unless extern weak
+  // or an absolute symbol reference. Other address spaces may have null as a
+  // valid address for a global, so we can't assume anything.
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(V))
-    return !GV->hasExternalWeakLinkage() &&
+    return !GV->isAbsoluteSymbolRef() && !GV->hasExternalWeakLinkage() &&
            GV->getType()->getAddressSpace() == 0;
 
   // A Load tagged with nonnull metadata is never null.
diff --git a/llvm/lib/IR/Globals.cpp b/llvm/lib/IR/Globals.cpp
index f8ac37f9fc3..7cad629eaa7 100644
--- a/llvm/lib/IR/Globals.cpp
+++ b/llvm/lib/IR/Globals.cpp
@@ -15,6 +15,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalValue.h"
@@ -222,6 +223,26 @@ GlobalObject *GlobalValue::getBaseObject() {
   return nullptr;
 }
 
+bool GlobalValue::isAbsoluteSymbolRef() const {
+  auto *GO = dyn_cast<GlobalObject>(this);
+  if (!GO)
+    return false;
+
+  return GO->getMetadata(LLVMContext::MD_absolute_symbol);
+}
+
+Optional<ConstantRange> GlobalValue::getAbsoluteSymbolRange() const {
+  auto *GO = dyn_cast<GlobalObject>(this);
+  if (!GO)
+    return None;
+
+  MDNode *MD = GO->getMetadata(LLVMContext::MD_absolute_symbol);
+  if (!MD)
+    return None;
+
+  return getConstantRangeFromMetadata(*MD);
+}
+
 //===----------------------------------------------------------------------===//
 // GlobalVariable Implementation
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/LLVMContext.cpp b/llvm/lib/IR/LLVMContext.cpp
index 0264283100c..dd66f144f04 100644
--- a/llvm/lib/IR/LLVMContext.cpp
+++ b/llvm/lib/IR/LLVMContext.cpp
@@ -57,6 +57,7 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) {
     {MD_loop, "llvm.loop"},
     {MD_type, "type"},
     {MD_section_prefix, "section_prefix"},
+    {MD_absolute_symbol, "absolute_symbol"},
   };
 
   for (auto &MDKind : MDKinds) {
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 19d1dd6e01b..14c95f94893 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -24,6 +24,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
+#include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Intrinsics.h"
@@ -1571,7 +1572,15 @@ bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
     return false;
 
   Imm = N;
-  return TM.getCodeModel() == CodeModel::Small;
+  if (N->getOpcode() != ISD::TargetGlobalAddress)
+    return TM.getCodeModel() == CodeModel::Small;
+
+  Optional<ConstantRange> CR =
+      cast<GlobalAddressSDNode>(N)->getGlobal()->getAbsoluteSymbolRange();
+  if (!CR)
+    return TM.getCodeModel() == CodeModel::Small;
+
+  return CR->getUnsignedMax().ult(1ull << 32);
 }
 
 bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base,
@@ -1710,10 +1719,39 @@ bool X86DAGToDAGISel::selectRelocImm(SDValue N, SDValue &Op) {
     return true;
   }
 
+  // Keep track of the original value type and whether this value was
+  // truncated. If we see a truncation from pointer type to VT that truncates
+  // bits that are known to be zero, we can use a narrow reference.
+  EVT VT = N.getValueType();
+  bool WasTruncated = false;
+  if (N.getOpcode() == ISD::TRUNCATE) {
+    WasTruncated = true;
+    N = N.getOperand(0);
+  }
+
   if (N.getOpcode() != X86ISD::Wrapper)
     return false;
 
-  Op = N.getOperand(0);
+  // We can only use non-GlobalValues as immediates if they were not truncated,
+  // as we do not have any range information. If we have a GlobalValue and the
+  // address was not truncated, we can select it as an operand directly.
+  unsigned Opc = N.getOperand(0)->getOpcode();
+  if (Opc != ISD::TargetGlobalAddress || !WasTruncated) {
+    Op = N.getOperand(0);
+    // We can only select the operand directly if we didn't have to look past a
+    // truncate.
+    return !WasTruncated;
+  }
+
+  // Check that the global's range fits into VT.
+  auto *GA = cast<GlobalAddressSDNode>(N.getOperand(0));
+  Optional<ConstantRange> CR = GA->getGlobal()->getAbsoluteSymbolRange();
+  if (!CR || CR->getUnsignedMax().uge(1ull << VT.getSizeInBits()))
+    return false;
+
+  // Okay, we can use a narrow reference.
+  Op = CurDAG->getTargetGlobalAddress(GA->getGlobal(), SDLoc(N), VT,
+                                      GA->getOffset(), GA->getTargetFlags());
   return true;
 }
 
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index baddf5b243c..75398c7ba7e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -13676,7 +13676,11 @@ static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
 }
 
 // Returns the appropriate wrapper opcode for a global reference.
-unsigned X86TargetLowering::getGlobalWrapperKind() const {
+unsigned X86TargetLowering::getGlobalWrapperKind(const GlobalValue *GV) const {
+  // References to absolute symbols are never PC-relative.
+  if (GV && GV->isAbsoluteSymbolRef())
+    return X86ISD::Wrapper;
+
   CodeModel::Model M = getTargetMachine().getCodeModel();
   if (Subtarget.isPICStyleRIPRel() &&
       (M == CodeModel::Small || M == CodeModel::Kernel))
@@ -13805,7 +13809,7 @@ SDValue X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV,
     Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, OpFlags);
   }
 
-  Result = DAG.getNode(getGlobalWrapperKind(), dl, PtrVT, Result);
+  Result = DAG.getNode(getGlobalWrapperKind(GV), dl, PtrVT, Result);
 
   // With PIC, the address is actually $g + Offset.
   if (isGlobalRelativeToPICBase(OpFlags)) {
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 4aeaf1bf510..728ebd5e23f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1115,7 +1115,7 @@ namespace llvm {
     SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
 
-    unsigned getGlobalWrapperKind() const;
+    unsigned getGlobalWrapperKind(const GlobalValue *GV = nullptr) const;
     SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerGlobalAddress(const GlobalValue *GV, const SDLoc &dl,
diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index 01185039b72..38036715a25 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -948,10 +948,10 @@ def i64immSExt32 : ImmLeaf<i64, [{ return isInt<32>(Imm); }]>;
 // Eventually, it would be nice to allow ConstantHoisting to merge constants
 // globally for potentially added savings.
 //
-def imm8_su : PatLeaf<(i8 imm), [{
+def imm8_su : PatLeaf<(i8 relocImm), [{
     return !shouldAvoidImmediateInstFormsForSize(N);
 }]>;
-def imm16_su : PatLeaf<(i16 imm), [{
+def imm16_su : PatLeaf<(i16 relocImm), [{
     return !shouldAvoidImmediateInstFormsForSize(N);
 }]>;
 def imm32_su : PatLeaf<(i32 relocImm), [{
diff --git a/llvm/lib/Target/X86/X86InstrShiftRotate.td b/llvm/lib/Target/X86/X86InstrShiftRotate.td
index 6303b6eaa30..e2be7353215 100644
--- a/llvm/lib/Target/X86/X86InstrShiftRotate.td
+++ b/llvm/lib/Target/X86/X86InstrShiftRotate.td
@@ -591,19 +591,20 @@ def ROR64rCL : RI<0xD3, MRM1r, (outs GR64:$dst), (ins GR64:$src1),
 
 def ROR8ri   : Ii8<0xC0, MRM1r, (outs GR8 :$dst), (ins GR8 :$src1, u8imm:$src2),
                    "ror{b}\t{$src2, $dst|$dst, $src2}",
-                   [(set GR8:$dst, (rotr GR8:$src1, (i8 imm:$src2)))], IIC_SR>;
+                   [(set GR8:$dst, (rotr GR8:$src1, (i8 relocImm:$src2)))],
+                   IIC_SR>;
 def ROR16ri  : Ii8<0xC1, MRM1r, (outs GR16:$dst), (ins GR16:$src1, u8imm:$src2),
                    "ror{w}\t{$src2, $dst|$dst, $src2}",
-                   [(set GR16:$dst, (rotr GR16:$src1, (i8 imm:$src2)))],
+                   [(set GR16:$dst, (rotr GR16:$src1, (i8 relocImm:$src2)))],
                    IIC_SR>, OpSize16;
 def ROR32ri  : Ii8<0xC1, MRM1r, (outs GR32:$dst), (ins GR32:$src1, u8imm:$src2),
                    "ror{l}\t{$src2, $dst|$dst, $src2}",
-                   [(set GR32:$dst, (rotr GR32:$src1, (i8 imm:$src2)))],
+                   [(set GR32:$dst, (rotr GR32:$src1, (i8 relocImm:$src2)))],
                    IIC_SR>, OpSize32;
 def ROR64ri  : RIi8<0xC1, MRM1r, (outs GR64:$dst),
                     (ins GR64:$src1, u8imm:$src2),
                     "ror{q}\t{$src2, $dst|$dst, $src2}",
-                    [(set GR64:$dst, (rotr GR64:$src1, (i8 imm:$src2)))],
+                    [(set GR64:$dst, (rotr GR64:$src1, (i8 relocImm:$src2)))],
                     IIC_SR>;
 
 // Rotate by 1
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index f1c490ca6b9..727ff70c3ff 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -92,6 +92,10 @@ unsigned char X86Subtarget::classifyGlobalReference(const GlobalValue *GV,
   if (TM.getCodeModel() == CodeModel::Large)
     return X86II::MO_NO_FLAG;
 
+  // Absolute symbols can be referenced directly.
+  if (GV && GV->isAbsoluteSymbolRef())
+    return X86II::MO_NO_FLAG;
+
   if (TM.shouldAssumeDSOLocal(M, GV))
     return classifyLocalReference(GV);
 
diff --git a/llvm/test/CodeGen/X86/absolute-bit-mask.ll b/llvm/test/CodeGen/X86/absolute-bit-mask.ll
new file mode 100644
index 00000000000..6e119494ac3
--- /dev/null
+++ b/llvm/test/CodeGen/X86/absolute-bit-mask.ll
@@ -0,0 +1,61 @@
+; RUN: llc < %s | FileCheck %s
+; RUN: llc -relocation-model=pic < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@bit_mask8 = external hidden global i8, !absolute_symbol !0
+@bit_mask32 = external hidden global i8, !absolute_symbol !1
+@bit_mask64 = external hidden global i8, !absolute_symbol !2
+
+declare void @f()
+
+define void @foo8(i8* %ptr) {
+  %load = load i8, i8* %ptr
+  ; CHECK: testb $bit_mask8, (%rdi)
+  %and = and i8 %load, ptrtoint (i8* @bit_mask8 to i8)
+  %icmp = icmp eq i8 %and, 0
+  br i1 %icmp, label %t, label %f
+
+t:
+  call void @f()
+  ret void
+
+f:
+  ret void
+}
+
+define void @foo32(i32* %ptr) {
+  %load = load i32, i32* %ptr
+  ; CHECK: testl $bit_mask32, (%rdi)
+  %and = and i32 %load, ptrtoint (i8* @bit_mask32 to i32)
+  %icmp = icmp eq i32 %and, 0
+  br i1 %icmp, label %t, label %f
+
+t:
+  call void @f()
+  ret void
+
+f:
+  ret void
+}
+
+define void @foo64(i64* %ptr) {
+  %load = load i64, i64* %ptr
+  ; CHECK: movabsq $bit_mask64, %rax
+  ; CHECK: testq (%rdi), %rax
+  %and = and i64 %load, ptrtoint (i8* @bit_mask64 to i64)
+  %icmp = icmp eq i64 %and, 0
+  br i1 %icmp, label %t, label %f
+
+t:
+  call void @f()
+  ret void
+
+f:
+  ret void
+}
+
+!0 = !{i64 0, i64 256}
+!1 = !{i64 0, i64 4294967296}
+!2 = !{i64 -1, i64 -1}
diff --git a/llvm/test/CodeGen/X86/absolute-bt.ll b/llvm/test/CodeGen/X86/absolute-bt.ll
new file mode 100644
index 00000000000..ffc16bc62ee
--- /dev/null
+++ b/llvm/test/CodeGen/X86/absolute-bt.ll
@@ -0,0 +1,51 @@
+; RUN: llc < %s | FileCheck %s
+; RUN: llc -relocation-model=pic < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@bit_mask8 = external hidden global i8, !absolute_symbol !0
+@bit_mask32 = external hidden global i8, !absolute_symbol !1
+@bit_mask64 = external hidden global i8, !absolute_symbol !2
+
+declare void @f()
+
+define void @foo32(i32* %ptr) {
+  %load = load i32, i32* %ptr
+  %and = and i32 %load, 31
+  %shl = shl i32 1, %and
+  %and2 = and i32 %shl, ptrtoint (i8* @bit_mask32 to i32)
+  ; CHECK: movl $bit_mask32, %eax
+  ; CHECK: btl %ecx, %eax
+  %icmp = icmp eq i32 %and2, 0
+  br i1 %icmp, label %t, label %f
+
+t:
+  call void @f()
+  ret void
+
+f:
+  ret void
+}
+
+define void @foo64(i64* %ptr) {
+  %load = load i64, i64* %ptr
+  %and = and i64 %load, 63
+  %shl = shl i64 1, %and
+  %and2 = and i64 %shl, ptrtoint (i8* @bit_mask64 to i64)
+  ; CHECK: movabsq $bit_mask64, %rax
+  ; CHECK: btq %rcx, %rax
+  %icmp = icmp eq i64 %and2, 0
+  br i1 %icmp, label %t, label %f
+
+t:
+  call void @f()
+  ret void
+
+f:
+  ret void
+}
+
+!0 = !{i64 0, i64 256}
+!1 = !{i64 0, i64 4294967296}
+!2 = !{i64 -1, i64 -1}
diff --git a/llvm/test/CodeGen/X86/absolute-constant.ll b/llvm/test/CodeGen/X86/absolute-constant.ll
new file mode 100644
index 00000000000..d93fb276c8e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/absolute-constant.ll
@@ -0,0 +1,28 @@
+; RUN: llc < %s | FileCheck %s
+; RUN: llc -relocation-model=pic < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = external global i8, align 1, !absolute_symbol !0
+
+define void @bar(i8* %x) {
+entry:
+  %0 = load i8, i8* %x, align 1
+  %conv = sext i8 %0 to i32
+  ; CHECK: testb $foo, (%rdi)
+  %and = and i32 %conv, sext (i8 ptrtoint (i8* @foo to i8) to i32)
+  %tobool = icmp eq i32 %and, 0
+  br i1 %tobool, label %if.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  tail call void (...) @xf()
+  br label %if.end
+
+if.end:                                           ; preds = %entry, %if.then
+  ret void
+}
+
+declare void @xf(...)
+
+!0 = !{i32 0, i32 256}
diff --git a/llvm/test/CodeGen/X86/absolute-rotate.ll b/llvm/test/CodeGen/X86/absolute-rotate.ll
new file mode 100644
index 00000000000..c0ecb82adc2
--- /dev/null
+++ b/llvm/test/CodeGen/X86/absolute-rotate.ll
@@ -0,0 +1,27 @@
+; RUN: llc < %s | FileCheck %s
+; RUN: llc -relocation-model=pic < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@align = external hidden global i8, !absolute_symbol !0
+
+declare void @f()
+
+define void @foo(i64 %val) {
+  %shr = lshr i64 %val, zext (i8 ptrtoint (i8* @align to i8) to i64)
+  %shl = shl i64 %val, zext (i8 sub (i8 64, i8 ptrtoint (i8* @align to i8)) to i64)
+  ; CHECK: rorq $align, %rdi
+  %ror = or i64 %shr, %shl
+  %cmp = icmp ult i64 %ror, 109
+  br i1 %cmp, label %t, label %f
+
+t:
+  call void @f()
+  ret void
+
+f:
+  ret void
+}
+
+!0 = !{i64 0, i64 256}
diff --git a/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll b/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
new file mode 100644
index 00000000000..43353e82270
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/nonnull-global.ll
@@ -0,0 +1,10 @@
+; RUN: opt -S -functionattrs %s | FileCheck %s
+
+@a = external global i8, !absolute_symbol !0
+
+; CHECK-NOT: define nonnull
+define i8* @foo() {
+  ret i8* @a
+}
+
+!0 = !{i64 0, i64 256}