Update malloc call creation code (AllocType is now the element type of the malloc, not the resulting type).

In getMallocArraySize(), fix bug in the case that array size is the product of 2 constants.

Extend isArrayMalloc() and getMallocArraySize() to handle case where malloc is used as char array.

Ensure that ArraySize in LowerAllocations::runOnBasicBlock() is correct type.

Extend Instruction::isSafeToSpeculativelyExecute() to handle malloc calls.

Add verification for malloc calls.

Reviewed by Dan Gohman.

llvm-svn: 82257

5 files changed, 132 insertions, 89 deletions

diff --git a/llvm/lib/Analysis/MallocHelper.cpp b/llvm/lib/Analysis/MallocHelper.cpp
index 60930d5169b..9c1671d3064 100644
--- a/llvm/lib/Analysis/MallocHelper.cpp
+++ b/llvm/lib/Analysis/MallocHelper.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Constants.h"
 #include "llvm/Instructions.h"
 #include "llvm/Module.h"
+#include "llvm/Analysis/ConstantFolding.h"
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
@@ -24,10 +25,6 @@ using namespace llvm;
 
 /// isMalloc - Returns true if the the value is either a malloc call or a
 /// bitcast of the result of a malloc call.
-bool llvm::isMalloc(Value* I) {
-  return extractMallocCall(I) || extractMallocCallFromBitCast(I);
-}
-
 bool llvm::isMalloc(const Value* I) {
   return extractMallocCall(I) || extractMallocCallFromBitCast(I);
 }
@@ -79,40 +76,42 @@ const CallInst* llvm::extractMallocCallFromBitCast(const Value* I) {
                                       : NULL;
 }
 
-static bool isArrayMallocHelper(const CallInst *CI) {
+static bool isArrayMallocHelper(const CallInst *CI, LLVMContext &Context,
+                                const TargetData* TD) {
   if (!CI)
     return false;
 
-  // Only identify array mallocs for mallocs with 1 bitcast use.  The unique 
-  // bitcast is needed to determine the type/size of the array allocation.
-  if (!CI->hasOneUse()) return false;
+  const Type* T = getMallocAllocatedType(CI);
 
-  for (Value::use_const_iterator UI = CI->use_begin(), E = CI->use_end();
-       UI != E; )
-    if (!isa<BitCastInst>(cast<Instruction>(*UI++)))
-      return false;
+  // We can only indentify an array malloc if we know the type of the malloc 
+  // call.
+  if (!T) return false;
 
-  // malloc arg
   Value* MallocArg = CI->getOperand(1);
-  // element size
-  const Type* T = getMallocAllocatedType(CI);
-  if (!T) return false;
   Constant *ElementSize = ConstantExpr::getSizeOf(T);
-  
+  ElementSize = ConstantExpr::getTruncOrBitCast(ElementSize, 
+                                                MallocArg->getType());
+  Constant *FoldedElementSize = ConstantFoldConstantExpression(
+                                       cast<ConstantExpr>(ElementSize), 
+                                       Context, TD);
+
+
   if (isa<ConstantExpr>(MallocArg))
-    return (MallocArg == ElementSize) ? false : true;
+    return (MallocArg != ElementSize);
 
   BinaryOperator *BI = dyn_cast<BinaryOperator>(MallocArg);
   if (!BI)
     return false;
 
-  if (BI->getOpcode() != Instruction::Mul)
-    return false;
-      
-  if (BI->getOperand(1) != ElementSize)
-    return false;
-        
-  return true;
+  if (BI->getOpcode() == Instruction::Mul)
+    // ArraySize * ElementSize
+    if (BI->getOperand(1) == ElementSize ||
+        (FoldedElementSize && BI->getOperand(1) == FoldedElementSize))
+      return true;
+
+  // TODO: Detect case where MallocArg mul has been transformed to shl.
+
+  return false;
 }
 
 /// isArrayMalloc - Returns the corresponding CallInst if the instruction 
@@ -125,14 +124,16 @@ static bool isArrayMallocHelper(const CallInst *CI) {
 /// Otherwise it returns NULL.
 /// The unique bitcast is needed to determine the type/size of the array
 /// allocation.
-CallInst* llvm::isArrayMalloc(Value* I) {
+CallInst* llvm::isArrayMalloc(Value* I, LLVMContext &Context,
+                              const TargetData* TD) {
   CallInst *CI = extractMallocCall(I);
-  return (isArrayMallocHelper(CI)) ? CI : NULL;
+  return (isArrayMallocHelper(CI, Context, TD)) ? CI : NULL;
 }
 
-const CallInst* llvm::isArrayMalloc(const Value* I) {
+const CallInst* llvm::isArrayMalloc(const Value* I, LLVMContext &Context,
+                                    const TargetData* TD) {
   const CallInst *CI = extractMallocCall(I);
-  return (isArrayMallocHelper(CI)) ? CI : NULL;
+  return (isArrayMallocHelper(CI, Context, TD)) ? CI : NULL;
 }
 
 /// getMallocType - Returns the PointerType resulting from the malloc call.
@@ -142,14 +143,24 @@ const PointerType* llvm::getMallocType(const CallInst* CI) {
   assert(isMalloc(CI) && "GetMallocType and not malloc call");
   
   const BitCastInst* BCI = NULL;
+  
+  // Determine if CallInst has a bitcast use.
+  for (Value::use_const_iterator UI = CI->use_begin(), E = CI->use_end();
+       UI != E; )
+    if ((BCI = dyn_cast<BitCastInst>(cast<Instruction>(*UI++))))
+      break;
+
+  // Malloc call has 1 bitcast use and no other uses, so type is the bitcast's
+  // destination type.
+  if (BCI && CI->hasOneUse())
+    return cast<PointerType>(BCI->getDestTy());
 
-  // Determine type only if there is only 1 bitcast use of CI.
-  if (CI->hasOneUse())
-    for (Value::use_const_iterator UI = CI->use_begin(), E = CI->use_end();
-         UI != E; )
-      BCI = dyn_cast<BitCastInst>(cast<Instruction>(*UI++));
+  // Malloc call was not bitcast, so the type is the malloc's return type, i8*.
+  if (!BCI)
+    return cast<PointerType>(CI->getType());
 
-  return BCI ? reinterpret_cast<const PointerType*>(BCI->getDestTy()) : NULL;
+  // Type could not be determined.
+  return NULL;
 }
 
 /// getMallocAllocatedType - Returns the Type allocated by malloc call. This
@@ -177,15 +188,16 @@ static bool isConstantOne(Value *val) {
 ///  1. The malloc call's allocated type cannot be determined.
 ///  2. IR wasn't created by a call to CallInst::CreateMalloc() with a non-NULL
 ///     ArraySize.
-Value* llvm::getMallocArraySize(CallInst* CI) {
+Value* llvm::getMallocArraySize(CallInst* CI, LLVMContext &Context,
+                                const TargetData* TD) {
   // Match CreateMalloc's use of constant 1 array-size for non-array mallocs.
-  if (!isArrayMalloc(CI))
+  if (!isArrayMalloc(CI, Context, TD))
     return ConstantInt::get(CI->getOperand(1)->getType(), 1);
 
   Value* MallocArg = CI->getOperand(1);
   assert(getMallocAllocatedType(CI) && "getMallocArraySize and no type");
   Constant *ElementSize = ConstantExpr::getSizeOf(getMallocAllocatedType(CI));
-  ElementSize = ConstantExpr::getTruncOrBitCast(cast<Constant>(ElementSize), 
+  ElementSize = ConstantExpr::getTruncOrBitCast(ElementSize, 
                                                 MallocArg->getType());
 
   Constant* CO = dyn_cast<Constant>(MallocArg);
@@ -195,10 +207,12 @@ Value* llvm::getMallocArraySize(CallInst* CI) {
       
   if (isConstantOne(ElementSize))
     return MallocArg;
+    
+  if (CO)
+    return CO->getOperand(0);
+    
+  // TODO: Detect case where MallocArg mul has been transformed to shl.
 
-  if (CO) 
-    return ConstantExpr::getUDiv(CO, ElementSize);
-  
   assert(BO && "getMallocArraySize not constant but not multiplication either");
   return BO->getOperand(0);
 }
diff --git a/llvm/lib/Transforms/Utils/LowerAllocations.cpp b/llvm/lib/Transforms/Utils/LowerAllocations.cpp
index cc1f6ea5e50..2df953c1499 100644
--- a/llvm/lib/Transforms/Utils/LowerAllocations.cpp
+++ b/llvm/lib/Transforms/Utils/LowerAllocations.cpp
@@ -108,8 +108,12 @@ bool LowerAllocations::runOnBasicBlock(BasicBlock &BB) {
   // Loop over all of the instructions, looking for malloc or free instructions
   for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ++I) {
     if (MallocInst *MI = dyn_cast<MallocInst>(I)) {
-      Value *MCast = CallInst::CreateMalloc(I, MI->getType(), IntPtrTy,
-                                            MI->getOperand(0));
+      Value *ArraySize = MI->getOperand(0);
+      if (ArraySize->getType() != IntPtrTy)
+        ArraySize = CastInst::CreateIntegerCast(ArraySize, IntPtrTy,
+                                                false /*ZExt*/, "", I);
+      Value *MCast = CallInst::CreateMalloc(I, IntPtrTy,
+                                            MI->getAllocatedType(), ArraySize);
 
       // Replace all uses of the old malloc inst with the cast inst
       MI->replaceAllUsesWith(MCast);
diff --git a/llvm/lib/VMCore/Instruction.cpp b/llvm/lib/VMCore/Instruction.cpp
index 815dd7e7bd9..efa80d16ef9 100644
--- a/llvm/lib/VMCore/Instruction.cpp
+++ b/llvm/lib/VMCore/Instruction.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Function.h"
 #include "llvm/Constants.h"
 #include "llvm/GlobalVariable.h"
+#include "llvm/Module.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/LeakDetector.h"
 using namespace llvm;
@@ -375,6 +376,27 @@ bool Instruction::isCommutative(unsigned op) {
   }
 }
 
+// Code here matches isMalloc from MallocHelper, which is not in VMCore.
+static bool isMalloc(const Value* I) {
+  const CallInst *CI = dyn_cast<CallInst>(I);
+  if (!CI) {
+  	const BitCastInst *BCI = dyn_cast<BitCastInst>(I);
+	if (!BCI) return false;
+
+    CI = dyn_cast<CallInst>(BCI->getOperand(0));
+  }
+
+  if (!CI) return false;
+
+  const Module* M = CI->getParent()->getParent()->getParent();
+  Constant *MallocFunc = M->getFunction("malloc");
+
+  if (CI->getOperand(0) != MallocFunc)
+    return false;
+
+  return true;
+}
+
 bool Instruction::isSafeToSpeculativelyExecute() const {
   for (unsigned i = 0, e = getNumOperands(); i != e; ++i)
     if (Constant *C = dyn_cast<Constant>(getOperand(i)))
@@ -400,7 +422,7 @@ bool Instruction::isSafeToSpeculativelyExecute() const {
   case Load: {
     if (cast<LoadInst>(this)->isVolatile())
       return false;
-    if (isa<AllocationInst>(getOperand(0)))
+    if (isa<AllocationInst>(getOperand(0)) || isMalloc(getOperand(0)))
       return true;
     if (GlobalVariable *GV = dyn_cast<GlobalVariable>(getOperand(0)))
       return !GV->hasExternalWeakLinkage();
diff --git a/llvm/lib/VMCore/Instructions.cpp b/llvm/lib/VMCore/Instructions.cpp
index bf0d0427f37..d479d9ac99c 100644
--- a/llvm/lib/VMCore/Instructions.cpp
+++ b/llvm/lib/VMCore/Instructions.cpp
@@ -460,19 +460,20 @@ static Value *checkArraySize(Value *Amt, const Type *IntPtrTy) {
 }
 
 static Value *createMalloc(Instruction *InsertBefore, BasicBlock *InsertAtEnd,
-                           const Type *AllocTy, const Type *IntPtrTy,
+                           const Type *IntPtrTy, const Type *AllocTy,
                            Value *ArraySize, const Twine &NameStr) {
   assert(((!InsertBefore && InsertAtEnd) || (InsertBefore && !InsertAtEnd)) &&
-         "createMalloc needs only InsertBefore or InsertAtEnd");
-  const PointerType *AllocPtrType = dyn_cast<PointerType>(AllocTy);
-  assert(AllocPtrType && "CreateMalloc passed a non-pointer allocation type");
-  
+         "createMalloc needs either InsertBefore or InsertAtEnd");
+
+  // malloc(type) becomes: 
+  //       bitcast (i8* malloc(typeSize)) to type*
+  // malloc(type, arraySize) becomes:
+  //       bitcast (i8 *malloc(typeSize*arraySize)) to type*
+  Value *AllocSize = ConstantExpr::getSizeOf(AllocTy);
+  AllocSize = ConstantExpr::getTruncOrBitCast(cast<Constant>(AllocSize),
+                                              IntPtrTy);
   ArraySize = checkArraySize(ArraySize, IntPtrTy);
 
-  // malloc(type) becomes i8 *malloc(size)
-  Value *AllocSize = ConstantExpr::getSizeOf(AllocPtrType->getElementType());
-  AllocSize = ConstantExpr::getTruncOrBitCast(cast<Constant>(AllocSize), 
-                                              IntPtrTy);
   if (!IsConstantOne(ArraySize)) {
     if (IsConstantOne(AllocSize)) {
       AllocSize = ArraySize;         // Operand * 1 = Operand
@@ -482,47 +483,41 @@ static Value *createMalloc(Instruction *InsertBefore, BasicBlock *InsertAtEnd,
       // Malloc arg is constant product of type size and array size
       AllocSize = ConstantExpr::getMul(Scale, cast<Constant>(AllocSize));
     } else {
-      Value *Scale = ArraySize;
-      if (Scale->getType() != IntPtrTy) {
-        if (InsertBefore)
-          Scale = CastInst::CreateIntegerCast(Scale, IntPtrTy, false /*ZExt*/,
-                                              "", InsertBefore);
-        else
-          Scale = CastInst::CreateIntegerCast(Scale, IntPtrTy, false /*ZExt*/,
-                                              "", InsertAtEnd);
-      }
       // Multiply type size by the array size...
       if (InsertBefore)
-        AllocSize = BinaryOperator::CreateMul(Scale, AllocSize,
-                                              "", InsertBefore);
+        AllocSize = BinaryOperator::CreateMul(ArraySize, AllocSize,
+                                              "mallocsize", InsertBefore);
       else
-        AllocSize = BinaryOperator::CreateMul(Scale, AllocSize,
-                                              "", InsertAtEnd);
+        AllocSize = BinaryOperator::CreateMul(ArraySize, AllocSize,
+                                              "mallocsize", InsertAtEnd);
     }
   }
 
+  assert(AllocSize->getType() == IntPtrTy && "malloc arg is wrong size");
   // Create the call to Malloc.
   BasicBlock* BB = InsertBefore ? InsertBefore->getParent() : InsertAtEnd;
   Module* M = BB->getParent()->getParent();
   const Type *BPTy = PointerType::getUnqual(Type::getInt8Ty(BB->getContext()));
   // prototype malloc as "void *malloc(size_t)"
-  Constant *MallocFunc = M->getOrInsertFunction("malloc", BPTy, 
-                                                IntPtrTy, NULL);
+  Constant *MallocF = M->getOrInsertFunction("malloc", BPTy, IntPtrTy, NULL);
+  if (!cast<Function>(MallocF)->doesNotAlias(0))
+    cast<Function>(MallocF)->setDoesNotAlias(0);
+  const PointerType *AllocPtrType = PointerType::getUnqual(AllocTy);
   CallInst *MCall = NULL;
-  if (InsertBefore) 
-    MCall = CallInst::Create(MallocFunc, AllocSize, NameStr, InsertBefore);
-  else
-    MCall = CallInst::Create(MallocFunc, AllocSize, NameStr, InsertAtEnd);
+  Value    *MCast = NULL;
+  if (InsertBefore) {
+    MCall = CallInst::Create(MallocF, AllocSize, "malloccall", InsertBefore);
+    // Create a cast instruction to convert to the right type...
+    MCast = new BitCastInst(MCall, AllocPtrType, NameStr, InsertBefore);
+  } else {
+    MCall = CallInst::Create(MallocF, AllocSize, "malloccall", InsertAtEnd);
+    // Create a cast instruction to convert to the right type...
+    MCast = new BitCastInst(MCall, AllocPtrType, NameStr);
+  }
   MCall->setTailCall();
-
-  // Create a cast instruction to convert to the right type...
   assert(MCall->getType() != Type::getVoidTy(BB->getContext()) &&
          "Malloc has void return type");
-  Value *MCast;
-  if (InsertBefore)
-    MCast = new BitCastInst(MCall, AllocPtrType, NameStr, InsertBefore);
-  else
-    MCast = new BitCastInst(MCall, AllocPtrType, NameStr);
+
   return MCast;
 }
 
@@ -532,11 +527,10 @@ static Value *createMalloc(Instruction *InsertBefore, BasicBlock *InsertAtEnd,
 ///    constant 1.
 /// 2. Call malloc with that argument.
 /// 3. Bitcast the result of the malloc call to the specified type.
-Value *CallInst::CreateMalloc(Instruction *InsertBefore,
-                              const Type *AllocTy, const Type *IntPtrTy,
-                              Value *ArraySize, const Twine &NameStr) {
-  return createMalloc(InsertBefore, NULL, AllocTy,
-                      IntPtrTy, ArraySize, NameStr);
+Value *CallInst::CreateMalloc(Instruction *InsertBefore, const Type *IntPtrTy,
+                              const Type *AllocTy, Value *ArraySize,
+                              const Twine &Name) {
+  return createMalloc(InsertBefore, NULL, IntPtrTy, AllocTy, ArraySize, Name);
 }
 
 /// CreateMalloc - Generate the IR for a call to malloc:
@@ -547,11 +541,10 @@ Value *CallInst::CreateMalloc(Instruction *InsertBefore,
 /// 3. Bitcast the result of the malloc call to the specified type.
 /// Note: This function does not add the bitcast to the basic block, that is the
 /// responsibility of the caller.
-Value *CallInst::CreateMalloc(BasicBlock *InsertAtEnd,
-                              const Type *AllocTy, const Type *IntPtrTy,
-                              Value *ArraySize, const Twine &NameStr) {
-  return createMalloc(NULL, InsertAtEnd, AllocTy, 
-                      IntPtrTy, ArraySize, NameStr);
+Value *CallInst::CreateMalloc(BasicBlock *InsertAtEnd, const Type *IntPtrTy,
+                              const Type *AllocTy, Value *ArraySize, 
+                              const Twine &Name) {
+  return createMalloc(NULL, InsertAtEnd, IntPtrTy, AllocTy, ArraySize, Name);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/VMCore/Verifier.cpp b/llvm/lib/VMCore/Verifier.cpp
index 140e6bd8b15..f1f6e2e199b 100644
--- a/llvm/lib/VMCore/Verifier.cpp
+++ b/llvm/lib/VMCore/Verifier.cpp
@@ -1143,6 +1143,16 @@ void Verifier::visitCallInst(CallInst &CI) {
   if (Function *F = CI.getCalledFunction())
     if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID())
       visitIntrinsicFunctionCall(ID, CI);
+
+  // Code here matches isMalloc from MallocHelper, which is not in VMCore.
+  const Module* M = CI.getParent()->getParent()->getParent();
+  Constant *MallocFunc = M->getFunction("malloc");
+
+  if (CI.getOperand(0) == MallocFunc) {
+    const PointerType *PTy =
+        PointerType::getUnqual(Type::getInt8Ty(CI.getParent()->getContext()));
+    Assert1(CI.getType() == PTy, "Malloc call must return i8*", &CI);
+  }
 }
 
 void Verifier::visitInvokeInst(InvokeInst &II) {