[asan] Add options -asan-detect-invalid-pointer-cmp and -asan-detect-invalid-pointer-sub options.

This is in preparation to a driver patch to add gcc 8's -fsanitize=pointer-compare and -fsanitize=pointer-subtract.
Disabled by default as this is still an experimental feature.

Reviewed By: morehouse, vitalybuka

Differential Revision: https://reviews.llvm.org/D59220

llvm-svn: 357157

2 files changed, 64 insertions, 6 deletions

diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index ec96d7b4615..f3bebf38ba2 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -275,6 +275,16 @@ static cl::opt<bool> ClInvalidPointerPairs(
     cl::desc("Instrument <, <=, >, >=, - with pointer operands"), cl::Hidden,
     cl::init(false));
 
+static cl::opt<bool> ClInvalidPointerCmp(
+    "asan-detect-invalid-pointer-cmp",
+    cl::desc("Instrument <, <=, >, >= with pointer operands"), cl::Hidden,
+    cl::init(false));
+
+static cl::opt<bool> ClInvalidPointerSub(
+    "asan-detect-invalid-pointer-sub",
+    cl::desc("Instrument - operations with pointer operands"), cl::Hidden,
+    cl::init(false));
+
 static cl::opt<unsigned> ClRealignStack(
     "asan-realign-stack",
     cl::desc("Realign stack to the value of this flag (power of two)"),
@@ -1408,11 +1418,24 @@ static bool isPointerOperand(Value *V) {
 // This is a rough heuristic; it may cause both false positives and
 // false negatives. The proper implementation requires cooperation with
 // the frontend.
-static bool isInterestingPointerComparisonOrSubtraction(Instruction *I) {
+static bool isInterestingPointerComparison(Instruction *I) {
   if (ICmpInst *Cmp = dyn_cast<ICmpInst>(I)) {
-    if (!Cmp->isRelational()) return false;
-  } else if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) {
-    if (BO->getOpcode() != Instruction::Sub) return false;
+    if (!Cmp->isRelational())
+      return false;
+  } else {
+    return false;
+  }
+  return isPointerOperand(I->getOperand(0)) &&
+         isPointerOperand(I->getOperand(1));
+}
+
+// This is a rough heuristic; it may cause both false positives and
+// false negatives. The proper implementation requires cooperation with
+// the frontend.
+static bool isInterestingPointerSubtraction(Instruction *I) {
+  if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) {
+    if (BO->getOpcode() != Instruction::Sub)
+      return false;
   } else {
     return false;
   }
@@ -2619,8 +2642,10 @@ bool AddressSanitizer::instrumentFunction(Function &F,
               continue; // We've seen this temp in the current BB.
           }
         }
-      } else if (ClInvalidPointerPairs &&
-                 isInterestingPointerComparisonOrSubtraction(&Inst)) {
+      } else if (((ClInvalidPointerPairs || ClInvalidPointerCmp) &&
+                  isInterestingPointerComparison(&Inst)) ||
+                 ((ClInvalidPointerPairs || ClInvalidPointerSub) &&
+                  isInterestingPointerSubtraction(&Inst))) {
         PointerComparisonsOrSubtracts.push_back(&Inst);
         continue;
       } else if (isa<MemIntrinsic>(Inst)) {
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
new file mode 100644
index 00000000000..3df73e54803
--- /dev/null
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-detect-invalid-pointer-pair.ll
@@ -0,0 +1,33 @@
+; RUN: opt < %s -asan -asan-detect-invalid-pointer-cmp -S \
+; RUN:     | FileCheck %s --check-prefixes=CMP,NOSUB,ALL
+; RUN: opt < %s -asan -asan-detect-invalid-pointer-sub -S \
+; RUN:     | FileCheck %s --check-prefixes=SUB,NOCMP,ALL
+; RUN: opt < %s -asan -asan-detect-invalid-pointer-pair -S \
+; RUN:     | FileCheck %s --check-prefixes=CMP,SUB,ALL
+; Support instrumentation of invalid pointer pair detection.
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @mycmp(i8* %p, i8* %q) sanitize_address {
+; ALL-LABEL: @mycmp
+; NOCMP-NOT: call void @__sanitizer_ptr_cmp
+; CMP: [[P:%[0-9A-Za-z]+]] = ptrtoint i8* %p to i64
+; CMP: [[Q:%[0-9A-Za-z]+]] = ptrtoint i8* %q to i64
+  %x = icmp ule i8* %p, %q
+; CMP: call void @__sanitizer_ptr_cmp(i64 [[P]], i64 [[Q]])
+  %y = zext i1 %x to i32
+  ret i32 %y
+}
+
+define i32 @mysub(i8* %p, i8* %q) sanitize_address {
+; ALL-LABEL: @mysub
+; NOSUB-NOT: call void @__sanitizer_ptr_sub
+; SUB: [[P:%[0-9A-Za-z]+]] = ptrtoint i8* %p to i64
+; SUB: [[Q:%[0-9A-Za-z]+]] = ptrtoint i8* %q to i64
+  %x = ptrtoint i8* %p to i64
+  %y = ptrtoint i8* %q to i64
+  %z = sub i64 %x, %y
+; SUB: call void @__sanitizer_ptr_sub(i64 [[P]], i64 [[Q]])
+  %w = trunc i64 %z to i32
+  ret i32 %w
+}