[asan] make calloc crash instead of returning 0 on overflow (controlled by the allocator_may_return_null flag)

llvm-svn: 190128

3 files changed, 21 insertions, 11 deletions

diff --git a/compiler-rt/lib/asan/asan_allocator2.cc b/compiler-rt/lib/asan/asan_allocator2.cc
index 5d13b91c331..ee28bd8f18d 100644
--- a/compiler-rt/lib/asan/asan_allocator2.cc
+++ b/compiler-rt/lib/asan/asan_allocator2.cc
@@ -636,7 +636,8 @@ void *asan_malloc(uptr size, StackTrace *stack) {
 }
 
 void *asan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
-  if (CallocShouldReturnNullDueToOverflow(size, nmemb)) return 0;
+  if (CallocShouldReturnNullDueToOverflow(size, nmemb))
+    return AllocatorReturnNull();
   void *ptr = Allocate(nmemb * size, 8, stack, FROM_MALLOC, false);
   // If the memory comes from the secondary allocator no need to clear it
   // as it comes directly from mmap.
diff --git a/compiler-rt/lib/asan/lit_tests/TestCases/allocator_returns_null.cc b/compiler-rt/lib/asan/lit_tests/TestCases/allocator_returns_null.cc
index a459f24e108..281debba4e2 100644
--- a/compiler-rt/lib/asan/lit_tests/TestCases/allocator_returns_null.cc
+++ b/compiler-rt/lib/asan/lit_tests/TestCases/allocator_returns_null.cc
@@ -8,6 +8,8 @@
 // RUN: ASAN_OPTIONS=allocator_may_return_null=1     %t malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mNULL
 // RUN: ASAN_OPTIONS=allocator_may_return_null=0 not %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-cCRASH
 // RUN: ASAN_OPTIONS=allocator_may_return_null=1     %t calloc 2>&1 | FileCheck %s --check-prefix=CHECK-cNULL
+// RUN: ASAN_OPTIONS=allocator_may_return_null=0 not %t calloc-overflow 2>&1 | FileCheck %s --check-prefix=CHECK-coCRASH
+// RUN: ASAN_OPTIONS=allocator_may_return_null=1     %t calloc-overflow 2>&1 | FileCheck %s --check-prefix=CHECK-coNULL
 // RUN: ASAN_OPTIONS=allocator_may_return_null=0 not %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-rCRASH
 // RUN: ASAN_OPTIONS=allocator_may_return_null=1     %t realloc 2>&1 | FileCheck %s --check-prefix=CHECK-rNULL
 // RUN: ASAN_OPTIONS=allocator_may_return_null=0 not %t realloc-after-malloc 2>&1 | FileCheck %s --check-prefix=CHECK-mrCRASH
@@ -32,6 +34,14 @@ int main(int argc, char **argv) {
     x = (char*)calloc(size / 4, 4);
   }
 
+  if (!strcmp(argv[1], "calloc-overflow")) {
+    fprintf(stderr, "calloc-overflow:\n");
+    volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
+    size_t kArraySize = 4096;
+    volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
+    x = (char*)calloc(kArraySize, kArraySize2);
+  }
+
   if (!strcmp(argv[1], "realloc")) {
     fprintf(stderr, "realloc:\n");
     x = (char*)realloc(0, size);
@@ -50,6 +60,8 @@ int main(int argc, char **argv) {
 // CHECK-mCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-cCRASH: calloc:
 // CHECK-cCRASH: AddressSanitizer's allocator is terminating the process
+// CHECK-coCRASH: calloc-overflow:
+// CHECK-coCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-rCRASH: realloc:
 // CHECK-rCRASH: AddressSanitizer's allocator is terminating the process
 // CHECK-mrCRASH: realloc-after-malloc:
@@ -59,6 +71,8 @@ int main(int argc, char **argv) {
 // CHECK-mNULL: x: (nil)
 // CHECK-cNULL: calloc:
 // CHECK-cNULL: x: (nil)
+// CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: x: (nil)
 // CHECK-rNULL: realloc:
 // CHECK-rNULL: x: (nil)
 // CHECK-mrNULL: realloc-after-malloc:
diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cc b/compiler-rt/lib/asan/tests/asan_noinst_test.cc
index 0d681caabc4..9d8a09c2a2e 100644
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cc
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cc
@@ -759,20 +759,15 @@ TEST(AddressSanitizerInterface, GetOwnershipStressTest) {
     free(pointers[i]);
 }
 
-TEST(AddressSanitizerInterface, CallocOverflow) {
-  size_t kArraySize = 4096;
-  volatile size_t kMaxSizeT = std::numeric_limits<size_t>::max();
-  volatile size_t kArraySize2 = kMaxSizeT / kArraySize + 10;
-  void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
-  EXPECT_EQ(0L, Ident(p));
-}
 
-TEST(AddressSanitizerInterface, CallocOverflow2) {
+TEST(AddressSanitizerInterface, CallocOverflow32) {
 #if SANITIZER_WORDSIZE == 32
   size_t kArraySize = 112;
   volatile size_t kArraySize2 = 43878406;
-  void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
-  EXPECT_EQ(0L, Ident(p));
+  void *p = 0;
+  EXPECT_DEATH(p = calloc(kArraySize, kArraySize2),
+               "allocator is terminating the process instead of returning 0");
+  assert(!p);
 #endif
 }