[analyzer] Discard malloc-overflow bug-report when a known size is malloc'ed.

This patch ignores malloc-overflow bug in two cases:
Case1:
x = a/b; where n < b
malloc (x*n); Then x*n will not overflow.

Case2:
x = a; // when 'a' is a known value.
malloc (x*n);

Also replaced isa with dyn_cast.

Reject multiplication by zero cases in MallocOverflowSecurityChecker
Currently MallocOverflowSecurityChecker does not catch cases like:
malloc(n * 0 * sizeof(int));

This patch rejects such cases.

Two test cases added. malloc-overflow2.c has an example inspired from a code
in linux kernel where the current checker flags a warning while it should not.

A patch by Aditya Kumar!

Differential Revision: http://reviews.llvm.org/D9924

llvm-svn: 248446

1 files changed, 36 insertions, 0 deletions

diff --git a/clang/test/Analysis/malloc-overflow2.c b/clang/test/Analysis/malloc-overflow2.c
new file mode 100644
index 00000000000..93f0239fe09
--- /dev/null
+++ b/clang/test/Analysis/malloc-overflow2.c
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -analyze -analyzer-checker=alpha.security.MallocOverflow,unix -verify %s
+
+typedef __typeof__(sizeof(int)) size_t;
+extern void *malloc(size_t);
+extern void free(void *ptr);
+
+void *malloc(unsigned long s);
+
+struct table {
+  int nentry;
+  unsigned *table;
+  unsigned offset_max;
+};
+
+static int table_build(struct table *t) {
+
+  t->nentry = ((t->offset_max >> 2) + 31) / 32;
+  t->table = (unsigned *)malloc(sizeof(unsigned) * t->nentry); // expected-warning {{the computation of the size of the memory allocation may overflow}}
+
+  int n;
+  n = 10000;
+  int *p = malloc(sizeof(int) * n); // no-warning
+
+  free(p);
+  return t->nentry;
+}
+
+static int table_build_1(struct table *t) {
+  t->nentry = (sizeof(struct table) * 2 + 31) / 32;
+  t->table = (unsigned *)malloc(sizeof(unsigned) * t->nentry); // no-warning
+  return t->nentry;
+}
+
+void *f(int n) {
+  return malloc(n * 0 * sizeof(int)); // expected-warning {{Call to 'malloc' has an allocation size of 0 bytes}}
+}