Fix a integer overflow in SimplifyCFG's look up table formation logic.

If the width is very large it gets truncated from uint64_t to uint32_t when
passed to TD->fitsInLegalInteger. The truncated value can fit in a register.
This manifested in massive memory usage or crashes (PR13946).

llvm-svn: 164784

2 files changed, 42 insertions, 0 deletions

diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 299c0596da3..065325b7c25 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -3414,6 +3414,10 @@ bool SwitchLookupTable::WouldFitInRegister(const TargetData *TD,
     return false;
   // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
   // are <= 15, we could try to narrow the type.
+
+  // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
+  if (TableSize >= UINT_MAX/IT->getBitWidth())
+    return false;
   return TD->fitsInLegalInteger(TableSize * IT->getBitWidth());
 }
 
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll
index ef9d51669d3..134ac4eeb14 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch_to_lookup_table.ll
@@ -231,3 +231,41 @@ lor.end:
 ; CHECK-NEXT: %lor.ext = zext i1 %1 to i32
 ; CHECK-NEXT: ret i32 %lor.ext
 }
+
+; PR13946
+define i32 @overflow(i32 %type) nounwind {
+entry:
+  switch i32 %type, label %sw.default [
+    i32 -2147483648, label %sw.bb
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 -2147483645, label %sw.bb3
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb:
+  br label %if.end
+
+sw.bb1:
+  br label %if.end
+
+sw.bb2:
+  br label %if.end
+
+sw.bb3:
+  br label %if.end
+
+sw.default:
+  br label %if.end
+
+if.else:
+  br label %if.end
+
+if.end:
+  %dirent_type.0 = phi i32 [ 3, %sw.default ], [ 6, %sw.bb3 ], [ 5, %sw.bb2 ], [ 0, %sw.bb1 ], [ 3, %sw.bb ], [ 0, %if.else ]
+  ret i32 %dirent_type.0
+; CHECK: define i32 @overflow
+; CHECK: switch
+; CHECK: phi
+}