[LSan] Make LSan allocator allocator_may_return_null compliant

Summary:
An attempt to reland D34786 (which caused bot failres on Mac), now with
properly intercepted operators new() and delete().

LSan allocator used to always return nullptr on too big allocation requests
(the definition of "too big" depends on platform and bitness), now it
follows policy configured by allocator_may_return_null flag

Reviewers: eugenis

Subscribers: llvm-commits

Differential Revision: https://reviews.llvm.org/D34845

llvm-svn: 306845

3 files changed, 169 insertions, 16 deletions

diff --git a/compiler-rt/lib/lsan/lsan_allocator.cc b/compiler-rt/lib/lsan/lsan_allocator.cc
index f54e953731b..6514aea6f60 100644
--- a/compiler-rt/lib/lsan/lsan_allocator.cc
+++ b/compiler-rt/lib/lsan/lsan_allocator.cc
@@ -74,7 +74,7 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
     size = 1;
   if (size > kMaxAllowedMallocSize) {
     Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", size);
-    return nullptr;
+    return Allocator::FailureHandler::OnBadRequest();
   }
   void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
   // Do not rely on the allocator to clear the memory (it's slow).
@@ -99,7 +99,7 @@ void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
   if (new_size > kMaxAllowedMallocSize) {
     Report("WARNING: LeakSanitizer failed to allocate %zu bytes\n", new_size);
     allocator.Deallocate(GetAllocatorCache(), p);
-    return nullptr;
+    return Allocator::FailureHandler::OnBadRequest();
   }
   p = allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
   RegisterAllocation(stack, p, new_size);
@@ -134,6 +134,8 @@ void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
 }
 
 void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) {
+  if (CheckForCallocOverflow(size, nmemb))
+    return Allocator::FailureHandler::OnBadRequest();
   size *= nmemb;
   return Allocate(stack, size, 1, true);
 }
diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cc b/compiler-rt/lib/lsan/lsan_interceptors.cc
index 49ccb33e2d3..7d514402ad4 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cc
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cc
@@ -70,7 +70,6 @@ INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
     CHECK(allocated < kCallocPoolSize);
     return mem;
   }
-  if (CheckForCallocOverflow(size, nmemb)) return nullptr;
   ENSURE_LSAN_INITED;
   GET_STACK_TRACE_MALLOC;
   return lsan_calloc(nmemb, size, stack);
@@ -199,6 +198,7 @@ INTERCEPTOR(int, mprobe, void *ptr) {
 }
 #endif // SANITIZER_INTERCEPT_MCHECK_MPROBE
 
+
 // TODO(alekseys): throw std::bad_alloc instead of dying on OOM.
 #define OPERATOR_NEW_BODY(nothrow)                         \
   ENSURE_LSAN_INITED;                                      \
@@ -207,22 +207,28 @@ INTERCEPTOR(int, mprobe, void *ptr) {
   if (!nothrow && UNLIKELY(!res)) DieOnFailure::OnOOM();\
   return res;
 
+#define OPERATOR_DELETE_BODY \
+  ENSURE_LSAN_INITED;        \
+  Deallocate(ptr);
+
+// On OS X it's not enough to just provide our own 'operator new' and
+// 'operator delete' implementations, because they're going to be in the runtime
+// dylib, and the main executable will depend on both the runtime dylib and
+// libstdc++, each of has its implementation of new and delete.
+// To make sure that C++ allocation/deallocation operators are overridden on
+// OS X we need to intercept them using their mangled names.
+#if !SANITIZER_MAC
+
 INTERCEPTOR_ATTRIBUTE
 void *operator new(size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
 void *operator new[](size_t size) { OPERATOR_NEW_BODY(false /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new(size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(true /*nothrow*/);
-}
+void *operator new(size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
 INTERCEPTOR_ATTRIBUTE
-void *operator new[](size_t size, std::nothrow_t const&) {
-  OPERATOR_NEW_BODY(true /*nothrow*/);
-}
-
-#define OPERATOR_DELETE_BODY \
-  ENSURE_LSAN_INITED;        \
-  Deallocate(ptr);
+void *operator new[](size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
 
 INTERCEPTOR_ATTRIBUTE
 void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
@@ -231,9 +237,31 @@ void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; }
 INTERCEPTOR_ATTRIBUTE
 void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; }
 INTERCEPTOR_ATTRIBUTE
-void operator delete[](void *ptr, std::nothrow_t const &) {
-  OPERATOR_DELETE_BODY;
-}
+void operator delete[](void *ptr, std::nothrow_t const &)
+{ OPERATOR_DELETE_BODY; }
+
+#else  // SANITIZER_MAC
+
+INTERCEPTOR(void *, _Znwm, size_t size)
+{ OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR(void *, _Znam, size_t size)
+{ OPERATOR_NEW_BODY(false /*nothrow*/); }
+INTERCEPTOR(void *, _ZnwmRKSt9nothrow_t, size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
+INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&)
+{ OPERATOR_NEW_BODY(true /*nothrow*/); }
+
+INTERCEPTOR(void, _ZdlPv, void *ptr)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdaPv, void *ptr)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdlPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+INTERCEPTOR(void, _ZdaPvRKSt9nothrow_t, void *ptr, std::nothrow_t const&)
+{ OPERATOR_DELETE_BODY; }
+
+#endif  // !SANITIZER_MAC
+
 
 ///// Thread initialization and finalization. /////
 
diff --git a/compiler-rt/test/lsan/TestCases/allocator_returns_null.cc b/compiler-rt/test/lsan/TestCases/allocator_returns_null.cc
new file mode 100644
index 00000000000..ab2c734e1e5
--- /dev/null
+++ b/compiler-rt/test/lsan/TestCases/allocator_returns_null.cc
@@ -0,0 +1,123 @@
+// Test the behavior of malloc/calloc/realloc/new when the allocation size is
+// more than LSan allocator's max allowed one.
+// By default (allocator_may_return_null=0) the process should crash.
+// With allocator_may_return_null=1 the allocator should return 0, except the
+// operator new(), which should crash anyway (operator new(std::nothrow) should
+// return nullptr, indeed).
+//
+// RUN: %clangxx_lsan -O0 %s -o %t
+// RUN: not %run %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-cCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t calloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-cNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t calloc-overflow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-coCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t calloc-overflow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-coNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-rCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t realloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-rNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mrCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t realloc-after-malloc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-mrNULL
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t new 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1 not %run %t new 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=0 not %run %t new-nothrow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nnCRASH
+// RUN: %env_lsan_opts=allocator_may_return_null=1     %run %t new-nothrow 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=CHECK-nnNULL
+
+#include <assert.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits>
+#include <new>
+
+int main(int argc, char **argv) {
+  // Disable stderr buffering. Needed on Windows.
+  setvbuf(stderr, NULL, _IONBF, 0);
+
+  assert(argc == 2);
+  const char *action = argv[1];
+  fprintf(stderr, "%s:\n", action);
+
+  // Use max of ASan and LSan allocator limits to cover both "lsan" and
+  // "lsan + asan" configs.
+  static const size_t kMaxAllowedMallocSizePlusOne =
+#if __LP64__ || defined(_WIN64)
+      (1ULL << 40) + 1;
+#else
+      (3UL << 30) + 1;
+#endif
+
+  void *x = 0;
+  if (!strcmp(action, "malloc")) {
+    x = malloc(kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "calloc")) {
+    x = calloc((kMaxAllowedMallocSizePlusOne / 4) + 1, 4);
+  } else if (!strcmp(action, "calloc-overflow")) {
+    volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+    size_t kArraySize = 4096;
+    volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+    x = calloc(kArraySize, kArraySize2);
+  } else if (!strcmp(action, "realloc")) {
+    x = realloc(0, kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "realloc-after-malloc")) {
+    char *t = (char*)malloc(100);
+    *t = 42;
+    x = realloc(t, kMaxAllowedMallocSizePlusOne);
+    assert(*t == 42);
+    free(t);
+  } else if (!strcmp(action, "new")) {
+    x = operator new(kMaxAllowedMallocSizePlusOne);
+  } else if (!strcmp(action, "new-nothrow")) {
+    x = operator new(kMaxAllowedMallocSizePlusOne, std::nothrow);
+  } else {
+    assert(0);
+  }
+
+  // The NULL pointer is printed differently on different systems, while (long)0
+  // is always the same.
+  fprintf(stderr, "x: %zu\n", (size_t)x);
+  free(x);
+
+  return x != 0;
+}
+
+// CHECK-mCRASH: malloc:
+// CHECK-mCRASH: Sanitizer's allocator is terminating the process
+// CHECK-cCRASH: calloc:
+// CHECK-cCRASH: Sanitizer's allocator is terminating the process
+// CHECK-coCRASH: calloc-overflow:
+// CHECK-coCRASH: Sanitizer's allocator is terminating the process
+// CHECK-rCRASH: realloc:
+// CHECK-rCRASH: Sanitizer's allocator is terminating the process
+// CHECK-mrCRASH: realloc-after-malloc:
+// CHECK-mrCRASH: Sanitizer's allocator is terminating the process
+// CHECK-nCRASH: new:
+// CHECK-nCRASH: Sanitizer's allocator is terminating the process
+// CHECK-nnCRASH: new-nothrow:
+// CHECK-nnCRASH: Sanitizer's allocator is terminating the process
+
+// CHECK-mNULL: malloc:
+// CHECK-mNULL: x: 0
+// CHECK-cNULL: calloc:
+// CHECK-cNULL: x: 0
+// CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: x: 0
+// CHECK-rNULL: realloc:
+// CHECK-rNULL: x: 0
+// CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: x: 0
+// CHECK-nnNULL: new-nothrow:
+// CHECK-nnNULL: x: 0