[sanitizer] improve the calloc overflow check (spotted by samsonov@)

llvm-svn: 173443

2 files changed, 12 insertions, 2 deletions

diff --git a/compiler-rt/lib/asan/tests/asan_noinst_test.cc b/compiler-rt/lib/asan/tests/asan_noinst_test.cc
index 95ad15e2cc3..278bde55d09 100644
--- a/compiler-rt/lib/asan/tests/asan_noinst_test.cc
+++ b/compiler-rt/lib/asan/tests/asan_noinst_test.cc
@@ -840,3 +840,12 @@ TEST(AddressSanitizerInterface, CallocOverflow) {
   void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
   EXPECT_EQ(0L, Ident(p));
 }
+
+TEST(AddressSanitizerInterface, CallocOverflow2) {
+#if SANITIZER_WORDSIZE == 32
+  size_t kArraySize = 112;
+  volatile size_t kArraySize2 = 43878406;
+  void *p = calloc(kArraySize, kArraySize2);  // Should return 0.
+  EXPECT_EQ(0L, Ident(p));
+#endif
+}
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
index 26baf73a1f3..88a3a1b2569 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cc
@@ -76,8 +76,9 @@ void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) {
 }
 
 bool CallocShouldReturnNullDueToOverflow(uptr size, uptr n) {
-  uptr mul = size * n;
-  return mul < size || mul < n;
+  if (!size) return false;
+  uptr max = (uptr)-1L;
+  return (max / size) < n;
 }
 
 }  // namespace __sanitizer