Fix dumping of characters with non-standard sizes

* Prevent dumping of characters in DumpDataExtractor() with
  item_byte_size bigger than 8 bytes. This case is not supported by the
  code and results in a crash because the code calls
  DataExtractor::GetMaxU64Bitfield() -> GetMaxU64() that asserts for
  byte size > 8 bytes.
* Teach DataExtractor::GetMaxU64(), GetMaxU32(), GetMaxS64() and
  GetMaxU64_unchecked() how to handle byte sizes that are not a multiple
  of 2. This allows DumpDataExtractor() to dump characters and booleans
  with item_byte_size in the interval of [1, 8] bytes. Values that are
  not a multiple of 2 would previously result in a crash because they
  were not handled by GetMaxU64().

llvm-svn: 315444

1 files changed, 117 insertions, 0 deletions

diff --git a/lldb/unittests/Core/DataExtractorTest.cpp b/lldb/unittests/Core/DataExtractorTest.cpp
index 213d5a7b43f..0267f6d1405 100644
--- a/lldb/unittests/Core/DataExtractorTest.cpp
+++ b/lldb/unittests/Core/DataExtractorTest.cpp
@@ -49,3 +49,120 @@ TEST(DataExtractorTest, PeekData) {
   EXPECT_EQ(buffer + 4, E.PeekData(4, 0));
   EXPECT_EQ(nullptr, E.PeekData(4, 1));
 }
+
+TEST(DataExtractorTest, GetMaxU64) {
+  uint8_t buffer[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
+  DataExtractor LE(buffer, sizeof(buffer), lldb::eByteOrderLittle,
+                   sizeof(void *));
+  DataExtractor BE(buffer, sizeof(buffer), lldb::eByteOrderBig, sizeof(void *));
+
+  lldb::offset_t offset;
+
+  // Check with the minimum allowed byte size.
+  offset = 0;
+  EXPECT_EQ(0x01U, LE.GetMaxU64(&offset, 1));
+  EXPECT_EQ(1U, offset);
+  offset = 0;
+  EXPECT_EQ(0x01U, BE.GetMaxU64(&offset, 1));
+  EXPECT_EQ(1U, offset);
+
+  // Check with a non-zero offset.
+  offset = 1;
+  EXPECT_EQ(0x0302U, LE.GetMaxU64(&offset, 2));
+  EXPECT_EQ(3U, offset);
+  offset = 1;
+  EXPECT_EQ(0x0203U, BE.GetMaxU64(&offset, 2));
+  EXPECT_EQ(3U, offset);
+
+  // Check with the byte size not being a multiple of 2.
+  offset = 0;
+  EXPECT_EQ(0x07060504030201U, LE.GetMaxU64(&offset, 7));
+  EXPECT_EQ(7U, offset);
+  offset = 0;
+  EXPECT_EQ(0x01020304050607U, BE.GetMaxU64(&offset, 7));
+  EXPECT_EQ(7U, offset);
+
+  // Check with the maximum allowed byte size.
+  offset = 0;
+  EXPECT_EQ(0x0807060504030201U, LE.GetMaxU64(&offset, 8));
+  EXPECT_EQ(8U, offset);
+  offset = 0;
+  EXPECT_EQ(0x0102030405060708U, BE.GetMaxU64(&offset, 8));
+  EXPECT_EQ(8U, offset);
+}
+
+TEST(DataExtractorTest, GetMaxS64) {
+  uint8_t buffer[] = {0x01, 0x02, 0x83, 0x04, 0x05, 0x06, 0x07, 0x08};
+  DataExtractor LE(buffer, sizeof(buffer), lldb::eByteOrderLittle,
+                   sizeof(void *));
+  DataExtractor BE(buffer, sizeof(buffer), lldb::eByteOrderBig, sizeof(void *));
+
+  lldb::offset_t offset;
+
+  // Check with the minimum allowed byte size.
+  offset = 0;
+  EXPECT_EQ(0x01, LE.GetMaxS64(&offset, 1));
+  EXPECT_EQ(1U, offset);
+  offset = 0;
+  EXPECT_EQ(0x01, BE.GetMaxS64(&offset, 1));
+  EXPECT_EQ(1U, offset);
+
+  // Check that sign extension works correctly.
+  offset = 0;
+  int64_t value = LE.GetMaxS64(&offset, 3);
+  EXPECT_EQ(0xffffffffff830201U, *reinterpret_cast<uint64_t *>(&value));
+  EXPECT_EQ(3U, offset);
+  offset = 2;
+  value = BE.GetMaxS64(&offset, 3);
+  EXPECT_EQ(0xffffffffff830405U, *reinterpret_cast<uint64_t *>(&value));
+  EXPECT_EQ(5U, offset);
+
+  // Check with the maximum allowed byte size.
+  offset = 0;
+  EXPECT_EQ(0x0807060504830201, LE.GetMaxS64(&offset, 8));
+  EXPECT_EQ(8U, offset);
+  offset = 0;
+  EXPECT_EQ(0x0102830405060708, BE.GetMaxS64(&offset, 8));
+  EXPECT_EQ(8U, offset);
+}
+
+TEST(DataExtractorTest, GetMaxU64_unchecked) {
+  uint8_t buffer[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
+  DataExtractor LE(buffer, sizeof(buffer), lldb::eByteOrderLittle,
+                   sizeof(void *));
+  DataExtractor BE(buffer, sizeof(buffer), lldb::eByteOrderBig, sizeof(void *));
+
+  lldb::offset_t offset;
+
+  // Check with the minimum allowed byte size.
+  offset = 0;
+  EXPECT_EQ(0x01U, LE.GetMaxU64_unchecked(&offset, 1));
+  EXPECT_EQ(1U, offset);
+  offset = 0;
+  EXPECT_EQ(0x01U, BE.GetMaxU64_unchecked(&offset, 1));
+  EXPECT_EQ(1U, offset);
+
+  // Check with a non-zero offset.
+  offset = 1;
+  EXPECT_EQ(0x0302U, LE.GetMaxU64_unchecked(&offset, 2));
+  EXPECT_EQ(3U, offset);
+  offset = 1;
+  EXPECT_EQ(0x0203U, BE.GetMaxU64_unchecked(&offset, 2));
+  EXPECT_EQ(3U, offset);
+
+  // Check with the byte size not being a multiple of 2.
+  offset = 0;
+  EXPECT_EQ(0x07060504030201U, LE.GetMaxU64_unchecked(&offset, 7));
+  EXPECT_EQ(7U, offset);
+  offset = 0;
+  EXPECT_EQ(0x01020304050607U, BE.GetMaxU64_unchecked(&offset, 7));
+  EXPECT_EQ(7U, offset);
+
+  // Check with the maximum allowed byte size.
+  offset = 0;
+  EXPECT_EQ(0x0807060504030201U, LE.GetMaxU64_unchecked(&offset, 8));
+  EXPECT_EQ(8U, offset);
+  offset = 0;
+  EXPECT_EQ(0x0102030405060708U, BE.GetMaxU64_unchecked(&offset, 8));
+  EXPECT_EQ(8U, offset);
+}