[tsan] make calloc crash instead of returning 0 on overflow (controlled by the allocator_may_return_null flag)

llvm-svn: 190135

5 files changed, 74 insertions, 4 deletions

diff --git a/compiler-rt/lib/tsan/lit_tests/allocator_returns_null.cc b/compiler-rt/lib/tsan/lit_tests/allocator_returns_null.cc
new file mode 100644
index 00000000000..4b5eb5504c2
--- /dev/null
+++ b/compiler-rt/lib/tsan/lit_tests/allocator_returns_null.cc
@@ -0,0 +1,64 @@
+// Test the behavior of malloc/calloc/realloc when the allocation size is huge.
+// By default (allocator_may_return_null=0) the process shoudl crash.
+// With allocator_may_return_null=1 the allocator should return 0.
+//
+// RUN: %clangxx_tsan -O0 %s -o %t
+// RUN: not %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: TSAN_OPTIONS=allocator_may_return_null=0 not %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mCRASH
+// RUN: TSAN_OPTIONS=allocator_may_return_null=0 not %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-cCRASH
+// RUN: TSAN_OPTIONS=allocator_may_return_null=0 not %t calloc-overflow 2>&1 | FileCheck %s --check-prefix=CHECK-coCRASH
+// RUN: TSAN_OPTIONS=allocator_may_return_null=0 not %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-rCRASH
+// RUN: TSAN_OPTIONS=allocator_may_return_null=0 not %t realloc-after-malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mrCRASH
+
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <assert.h>
+#include <limits>
+int main(int argc, char **argv) {
+  volatile size_t size = std::numeric_limits<size_t>::max() - 10000;
+  assert(argc == 2);
+  char *x = 0;
+  if (!strcmp(argv[1], "malloc")) {
+    fprintf(stderr, "malloc:\n");
+    x = (char*)malloc(size);
+  }
+  if (!strcmp(argv[1], "calloc")) {
+    fprintf(stderr, "calloc:\n");
+    x = (char*)calloc(size / 4, 4);
+  }
+
+  if (!strcmp(argv[1], "calloc-overflow")) {
+    fprintf(stderr, "calloc-overflow:\n");
+    volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+    size_t kArraySize = 4096;
+    volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+    x = (char*)calloc(kArraySize, kArraySize2);
+  }
+
+  if (!strcmp(argv[1], "realloc")) {
+    fprintf(stderr, "realloc:\n");
+    x = (char*)realloc(0, size);
+  }
+  if (!strcmp(argv[1], "realloc-after-malloc")) {
+    fprintf(stderr, "realloc-after-malloc:\n");
+    char *t = (char*)malloc(100);
+    *t = 42;
+    x = (char*)realloc(t, size);
+    assert(*t == 42);
+  }
+  fprintf(stderr, "x: %p\n", x);
+  return x != 0;
+}
+// CHECK-mCRASH: malloc:
+// CHECK-mCRASH: ThreadSanitizer's allocator is terminating the process
+// CHECK-cCRASH: calloc:
+// CHECK-cCRASH: ThreadSanitizer's allocator is terminating the process
+// CHECK-coCRASH: calloc-overflow:
+// CHECK-coCRASH: ThreadSanitizer's allocator is terminating the process
+// CHECK-rCRASH: realloc:
+// CHECK-rCRASH: ThreadSanitizer's allocator is terminating the process
+// CHECK-mrCRASH: realloc-after-malloc:
+// CHECK-mrCRASH: ThreadSanitizer's allocator is terminating the process
+
diff --git a/compiler-rt/lib/tsan/lit_tests/malloc_overflow.cc b/compiler-rt/lib/tsan/lit_tests/malloc_overflow.cc
index 19423c5f93f..8fd12dff1d0 100644
--- a/compiler-rt/lib/tsan/lit_tests/malloc_overflow.cc
+++ b/compiler-rt/lib/tsan/lit_tests/malloc_overflow.cc
@@ -1,4 +1,7 @@
-// RUN: %clangxx_tsan -O1 %s -o %t && %t 2>&1 | FileCheck %s
+// XFAIL: *
+// FIXME: https://code.google.com/p/thread-sanitizer/issues/detail?id=29
+// RUN: %clangxx_tsan -O1 %s -o %t
+// RUN: TSAN_OPTIONS=allocator_may_return_null=1 %t 2>&1 | FileCheck %s
 #include <stdio.h>
 #include <stdlib.h>
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
index 4af8dcb95e3..d43276e35cf 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc
@@ -443,7 +443,8 @@ TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
   if (cur_thread()->in_symbolizer)
     return __libc_calloc(size, n);
-  if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n)) return 0;
+  if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n))
+    return AllocatorReturnNull();
   void *p = 0;
   {
     SCOPED_INTERCEPTOR_RAW(calloc, size, n);
diff --git a/compiler-rt/lib/tsan/rtl/tsan_mman.cc b/compiler-rt/lib/tsan/rtl/tsan_mman.cc
index 07fd5890061..957479039b4 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_mman.cc
+++ b/compiler-rt/lib/tsan/rtl/tsan_mman.cc
@@ -104,7 +104,7 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) {
 void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) {
   CHECK_GT(thr->in_rtl, 0);
   if ((sz >= (1ull << 40)) || (align >= (1ull << 40)))
-    return 0;
+    return AllocatorReturnNull();
   void *p = allocator()->Allocate(&thr->alloc_cache, sz, align);
   if (p == 0)
     return 0;
diff --git a/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc b/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc
index 0961d2b75d1..e1ad7ac51ad 100644
--- a/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc
+++ b/compiler-rt/lib/tsan/tests/unit/tsan_mman_test.cc
@@ -164,7 +164,9 @@ TEST(Mman, CallocOverflow) {
   size_t kArraySize = 4096;
   volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
   volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
-  volatile void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
+  volatile void *p = NULL;
+  EXPECT_DEATH(p = calloc(kArraySize, kArraySize2),
+               "allocator is terminating the process instead of returning 0");
   EXPECT_EQ(0L, p);
 }