<rdar://problem/12725746>

Providing data formatters for char16_t* and char32_t* C++11-style Unicode strings
Using this chance to refactor the UTF data reader used for data formatters for added generality
Added a relevant test case

llvm-svn: 172119

6 files changed, 344 insertions, 126 deletions

diff --git a/lldb/include/lldb/Core/CXXFormatterFunctions.h b/lldb/include/lldb/Core/CXXFormatterFunctions.h
index 0b20f1d8249..3373b9ccb32 100644
--- a/lldb/include/lldb/Core/CXXFormatterFunctions.h
+++ b/lldb/include/lldb/Core/CXXFormatterFunctions.h
@@ -40,6 +40,15 @@ namespace lldb_private {
                               const char* selector,
                               const char* key);
         
+        bool
+        Char16StringSummaryProvider (ValueObject& valobj, Stream& stream); // char16_t*
+        
+        bool
+        Char32StringSummaryProvider (ValueObject& valobj, Stream& stream); // char32_t*
+        
+        bool
+        WCharStringSummaryProvider (ValueObject& valobj, Stream& stream); // wchar_t*
+        
         template<bool name_entries>
         bool
         NSDictionarySummaryProvider (ValueObject& valobj, Stream& stream);
diff --git a/lldb/source/Core/CXXFormatterFunctions.cpp b/lldb/source/Core/CXXFormatterFunctions.cpp
index 3dc48c560f5..defc2c1d0c0 100644
--- a/lldb/source/Core/CXXFormatterFunctions.cpp
+++ b/lldb/source/Core/CXXFormatterFunctions.cpp
@@ -135,6 +135,151 @@ lldb_private::formatters::CallSelectorOnObject (ValueObject &valobj,
     return valobj_sp;
 }
 
+template<typename SourceDataType, ConversionResult (*ConvertFunction) (const SourceDataType**,
+                                                                       const SourceDataType*,
+                                                                       UTF8**,
+                                                                       UTF8*,
+                                                                       ConversionFlags)>
+static bool
+ReadUTFBufferAndDumpToStream (uint64_t location,
+                              const ProcessSP& process_sp,
+                              Stream& stream,
+                              bool want_at = true,
+                              bool want_quotes = true)
+{
+    if (location == 0 || location == LLDB_INVALID_ADDRESS)
+        return false;
+    if (!process_sp)
+        return false;
+
+    const int origin_encoding = 8*sizeof(SourceDataType);
+    if (origin_encoding != 8 && origin_encoding != 16 && origin_encoding != 32)
+        return false;
+    // if not UTF8, I need a conversion function to return proper UTF8
+    if (origin_encoding != 8 && !ConvertFunction)
+        return false;
+    const int bufferSPSize = 1024;
+    const int sourceSize = (bufferSPSize / (origin_encoding >> 2));
+
+    Error error;
+    lldb::DataBufferSP buffer_sp(new DataBufferHeap(bufferSPSize,0));
+    size_t data_read = process_sp->ReadMemoryFromInferior(location, (char*)buffer_sp->GetBytes(), bufferSPSize, error);
+    if (error.Fail())
+    {
+        stream.Printf("unable to read data");
+        return true;
+    }
+    else
+    {
+        if (want_at)
+            stream.Printf("@");
+        if (want_quotes)
+            stream.Printf("\"");
+    }
+    if (data_read)
+    {
+        SourceDataType *data_ptr = (SourceDataType*)buffer_sp->GetBytes();
+        SourceDataType *data_end_ptr = data_ptr + sourceSize;
+        
+        while (data_ptr < data_end_ptr)
+        {
+            if (!*data_ptr)
+            {
+                data_end_ptr = data_ptr;
+                break;
+            }
+            data_ptr++;
+        }
+        
+        *data_ptr = 0;
+        data_ptr = (SourceDataType*)buffer_sp->GetBytes();
+        
+        lldb::DataBufferSP utf8_data_buffer_sp(new DataBufferHeap(bufferSPSize,0));
+        UTF8* utf8_data_ptr = (UTF8*)utf8_data_buffer_sp->GetBytes();
+        UTF8* utf8_data_end_ptr = utf8_data_ptr + bufferSPSize;
+        
+        if (ConvertFunction)
+            ConvertFunction ( (const SourceDataType**)&data_ptr, data_end_ptr, &utf8_data_ptr, utf8_data_end_ptr, lenientConversion );
+        else
+        {
+            // just copy the pointers - the cast is necessary to make the compiler happy
+            // but this should only happen if we are reading UTF8 data
+            utf8_data_ptr = (UTF8*)data_ptr;
+            utf8_data_end_ptr = (UTF8*)data_end_ptr;
+        }
+        
+        utf8_data_ptr = (UTF8*)utf8_data_buffer_sp->GetBytes();
+        for (;utf8_data_ptr != utf8_data_end_ptr; utf8_data_ptr++)
+        {
+            if (!*utf8_data_ptr)
+                break;
+            stream.Printf("%c",*utf8_data_ptr);
+        }
+        if (want_quotes)
+            stream.Printf("\"");
+        return true;
+    }
+    if (want_quotes)
+        stream.Printf("\"");
+    return true;
+}
+
+bool
+lldb_private::formatters::Char16StringSummaryProvider (ValueObject& valobj, Stream& stream)
+{
+    ProcessSP process_sp = valobj.GetProcessSP();
+    if (!process_sp)
+        return false;
+    
+    lldb::addr_t valobj_addr = valobj.GetValueAsUnsigned(0);
+    
+    if (!valobj_addr)
+        return false;
+    
+    if (!ReadUTFBufferAndDumpToStream<UTF16, ConvertUTF16toUTF8>(valobj_addr,
+                                                                 process_sp,
+                                                                 stream,
+                                                                 false, // no @ sign for C++
+                                                                 true)) // but use quotes
+    {
+        stream.Printf("Summary Unavailable");
+        return true;
+    }
+
+    return true;
+}
+
+bool
+lldb_private::formatters::Char32StringSummaryProvider (ValueObject& valobj, Stream& stream)
+{
+    ProcessSP process_sp = valobj.GetProcessSP();
+    if (!process_sp)
+        return false;
+    
+    lldb::addr_t valobj_addr = valobj.GetValueAsUnsigned(0);
+    
+    if (!valobj_addr)
+        return false;
+    
+    if (!ReadUTFBufferAndDumpToStream<UTF32, ConvertUTF32toUTF8>(valobj_addr,
+                                                                 process_sp,
+                                                                 stream,
+                                                                 false, // no @ sign for C++
+                                                                 true)) // but use quotes
+    {
+        stream.Printf("Summary Unavailable");
+        return true;
+    }
+    
+    return true;
+}
+
+bool
+lldb_private::formatters::WCharStringSummaryProvider (ValueObject& valobj, Stream& stream)
+{
+    return false;
+}
+
 template<bool name_entries>
 bool
 lldb_private::formatters::NSDictionarySummaryProvider (ValueObject& valobj, Stream& stream)
@@ -476,62 +621,6 @@ lldb_private::formatters::NSNumberSummaryProvider (ValueObject& valobj, Stream&
     }
 }
 
-static bool
-ReadUTFBufferAndDumpToStream (uint64_t location,
-                              const ProcessSP& process_sp,
-                              Stream& stream)
-{
-    Error error;
-    lldb::DataBufferSP buffer_sp(new DataBufferHeap(1024,0));
-    size_t data_read = process_sp->ReadMemoryFromInferior(location, (char*)buffer_sp->GetBytes(), 1024, error);
-    if (error.Fail())
-    {
-        stream.Printf("unable to read data");
-        return true;
-    }
-    else
-        stream.Printf("@\"");
-    if (data_read)
-    {
-        UTF16 *data_ptr = (UTF16*)buffer_sp->GetBytes();
-        UTF16 *data_end_ptr = data_ptr + 256;
-        
-        while (data_ptr < data_end_ptr)
-        {
-            if (!*data_ptr)
-            {
-                data_end_ptr = data_ptr;
-                break;
-            }
-            data_ptr++;
-        }
-        
-        *data_ptr = 0;
-        data_ptr = (UTF16*)buffer_sp->GetBytes();
-        
-        lldb::DataBufferSP utf8_data_buffer_sp(new DataBufferHeap(1024,0));
-        UTF8* utf8_data_ptr = (UTF8*)utf8_data_buffer_sp->GetBytes();
-        UTF8* utf8_data_end_ptr = utf8_data_ptr + 1024;
-        
-        ConvertUTF16toUTF8	(	(const UTF16**)&data_ptr,
-                             data_end_ptr,
-                             &utf8_data_ptr,
-                             utf8_data_end_ptr,
-                             lenientConversion);
-        utf8_data_ptr = (UTF8*)utf8_data_buffer_sp->GetBytes();
-        for (;utf8_data_ptr != utf8_data_end_ptr; utf8_data_ptr++)
-        {
-            if (!*utf8_data_ptr)
-                break;
-            stream.Printf("%c",*utf8_data_ptr);
-        }
-        stream.Printf("\"");
-        return true;
-    }
-    stream.Printf("\"");
-    return true;
-}
-
 bool
 lldb_private::formatters::NSStringSummaryProvider (ValueObject& valobj, Stream& stream)
 {
@@ -597,7 +686,7 @@ lldb_private::formatters::NSStringSummaryProvider (ValueObject& valobj, Stream&
         if (error.Fail())
             return false;
         if (has_explicit_length and is_unicode)
-            return ReadUTFBufferAndDumpToStream (location, process_sp, stream);
+            return ReadUTFBufferAndDumpToStream<UTF16,ConvertUTF16toUTF8> (location, process_sp, stream);
         else
         {
             location++;
@@ -640,12 +729,12 @@ lldb_private::formatters::NSStringSummaryProvider (ValueObject& valobj, Stream&
             if (error.Fail())
                 return false;
         }
-        return ReadUTFBufferAndDumpToStream (location, process_sp, stream);
+        return ReadUTFBufferAndDumpToStream<UTF16,ConvertUTF16toUTF8> (location, process_sp, stream);
     }
     else if (is_special)
     {
         uint64_t location = valobj_addr + (ptr_size == 8 ? 12 : 8);
-        return ReadUTFBufferAndDumpToStream (location, process_sp, stream);
+        return ReadUTFBufferAndDumpToStream<UTF16,ConvertUTF16toUTF8> (location, process_sp, stream);
     }
     else if (is_inline)
     {
diff --git a/lldb/source/Core/FormatManager.cpp b/lldb/source/Core/FormatManager.cpp
index 22dff065445..fe1e6ec05ed 100644
--- a/lldb/source/Core/FormatManager.cpp
+++ b/lldb/source/Core/FormatManager.cpp
@@ -719,6 +719,63 @@ FormatManager::FormatManager() :
     EnableCategory(m_system_category_name,CategoryMap::Last);
 }
 
+static void
+AddStringSummary(TypeCategoryImpl::SharedPointer category_sp,
+                 const char* string,
+                 ConstString type_name,
+                 TypeSummaryImpl::Flags flags)
+{
+    lldb::TypeSummaryImplSP summary_sp(new StringSummaryFormat(flags,
+                                                               string));
+    category_sp->GetSummaryNavigator()->Add(type_name,
+                                            summary_sp);
+}
+
+#ifndef LLDB_DISABLE_PYTHON
+static void
+AddScriptSummary(TypeCategoryImpl::SharedPointer category_sp,
+                 const char* funct_name,
+                 ConstString type_name,
+                 TypeSummaryImpl::Flags flags)
+{
+    
+    std::string code("     ");
+    code.append(funct_name).append("(valobj,internal_dict)");
+    
+    lldb::TypeSummaryImplSP summary_sp(new ScriptSummaryFormat(flags,
+                                                               funct_name,
+                                                               code.c_str()));
+    category_sp->GetSummaryNavigator()->Add(type_name,
+                                            summary_sp);
+}
+#endif
+
+#ifndef LLDB_DISABLE_PYTHON
+static void
+AddCXXSummary (TypeCategoryImpl::SharedPointer category_sp,
+               CXXFunctionSummaryFormat::Callback funct,
+               const char* description,
+               ConstString type_name,
+               TypeSummaryImpl::Flags flags)
+{
+    lldb::TypeSummaryImplSP summary_sp(new CXXFunctionSummaryFormat(flags,funct,description));
+    category_sp->GetSummaryNavigator()->Add(type_name,
+                                            summary_sp);
+}
+#endif
+
+#ifndef LLDB_DISABLE_PYTHON
+static void AddCXXSynthetic  (TypeCategoryImpl::SharedPointer category_sp,
+                              CXXSyntheticChildren::CreateFrontEndCallback generator,
+                              const char* description,
+                              ConstString type_name,
+                              TypeSyntheticImpl::Flags flags)
+{
+    lldb::SyntheticChildrenSP synth_sp(new CXXSyntheticChildren(flags,description,generator));
+    category_sp->GetSyntheticNavigator()->Add(type_name,synth_sp);
+}
+#endif
+
 void
 FormatManager::LoadSTLFormatters()
 {
@@ -839,14 +896,17 @@ FormatManager::LoadLibcxxFormatters()
 void
 FormatManager::LoadSystemFormatters()
 {
-    lldb::TypeSummaryImplSP string_format(new StringSummaryFormat(TypeSummaryImpl::Flags().SetCascades(false)
-                                                                  .SetSkipPointers(true)
-                                                                  .SetSkipReferences(false)
-                                                                  .SetDontShowChildren(true)
-                                                                  .SetDontShowValue(false)
-                                                                  .SetShowMembersOneLiner(false)
-                                                                  .SetHideItemNames(false),
-                                                                  "${var%s}"));
+    
+    TypeSummaryImpl::Flags string_flags;
+    string_flags.SetCascades(false)
+    .SetSkipPointers(true)
+    .SetSkipReferences(false)
+    .SetDontShowChildren(true)
+    .SetDontShowValue(false)
+    .SetShowMembersOneLiner(false)
+    .SetHideItemNames(false);
+    
+    lldb::TypeSummaryImplSP string_format(new StringSummaryFormat(string_flags, "${var%s}"));
     
     
     lldb::TypeSummaryImplSP string_array_format(new StringSummaryFormat(TypeSummaryImpl::Flags().SetCascades(false)
@@ -876,64 +936,14 @@ FormatManager::LoadSystemFormatters()
                                                                    "${var%O}"));
     
     sys_category_sp->GetSummaryNavigator()->Add(ConstString("OSType"), ostype_summary);
-}
-
-static void
-AddStringSummary(TypeCategoryImpl::SharedPointer category_sp,
-                 const char* string,
-                 ConstString type_name,
-                 TypeSummaryImpl::Flags flags)
-{
-    lldb::TypeSummaryImplSP summary_sp(new StringSummaryFormat(flags,
-                                                               string));
-    category_sp->GetSummaryNavigator()->Add(type_name,
-                                            summary_sp);
-}
-
-#ifndef LLDB_DISABLE_PYTHON
-static void
-AddScriptSummary(TypeCategoryImpl::SharedPointer category_sp,
-                 const char* funct_name,
-                 ConstString type_name,
-                 TypeSummaryImpl::Flags flags)
-{
     
-    std::string code("     ");
-    code.append(funct_name).append("(valobj,internal_dict)");
+    // FIXME because of a bug in the FormatNavigator we need to add a summary for both X* and const X* (<rdar://problem/12717717>)
+    AddCXXSummary(sys_category_sp, lldb_private::formatters::Char16StringSummaryProvider, "char16_t * summary provider", ConstString("char16_t *"), string_flags);
+    AddCXXSummary(sys_category_sp, lldb_private::formatters::Char16StringSummaryProvider, "char16_t * summary provider", ConstString("const char16_t *"), string_flags);
     
-    lldb::TypeSummaryImplSP summary_sp(new ScriptSummaryFormat(flags,
-                                                               funct_name,
-                                                               code.c_str()));
-    category_sp->GetSummaryNavigator()->Add(type_name,
-                                            summary_sp);
-}
-#endif
-
-#ifndef LLDB_DISABLE_PYTHON
-static void
-AddCXXSummary (TypeCategoryImpl::SharedPointer category_sp,
-               CXXFunctionSummaryFormat::Callback funct,
-               const char* description,
-               ConstString type_name,
-               TypeSummaryImpl::Flags flags)
-{
-    lldb::TypeSummaryImplSP summary_sp(new CXXFunctionSummaryFormat(flags,funct,description));
-    category_sp->GetSummaryNavigator()->Add(type_name,
-                                            summary_sp);
+    AddCXXSummary(sys_category_sp, lldb_private::formatters::Char32StringSummaryProvider, "char32_t * summary provider", ConstString("char32_t *"), string_flags);
+    AddCXXSummary(sys_category_sp, lldb_private::formatters::Char32StringSummaryProvider, "char32_t * summary provider", ConstString("const char32_t *"), string_flags);
 }
-#endif
-
-#ifndef LLDB_DISABLE_PYTHON
-static void AddCXXSynthetic  (TypeCategoryImpl::SharedPointer category_sp,
-                              CXXSyntheticChildren::CreateFrontEndCallback generator,
-                              const char* description,
-                              ConstString type_name,
-                              TypeSyntheticImpl::Flags flags)
-{
-    lldb::SyntheticChildrenSP synth_sp(new CXXSyntheticChildren(flags,description,generator));
-    category_sp->GetSyntheticNavigator()->Add(type_name,synth_sp);
-}
-#endif
 
 void
 FormatManager::LoadObjCFormatters()
@@ -960,10 +970,10 @@ FormatManager::LoadObjCFormatters()
 #ifndef LLDB_DISABLE_PYTHON
     // we need to skip pointers here since we are special casing a SEL* when retrieving its value
     objc_flags.SetSkipPointers(true);
-    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<false>, "SEL summary", ConstString("SEL"), objc_flags);
-    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<false>, "SEL summary", ConstString("struct objc_selector"), objc_flags);
-    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<false>, "SEL summary", ConstString("objc_selector"), objc_flags);
-    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<true>, "SEL summary", ConstString("objc_selector *"), objc_flags);
+    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<false>, "SEL summary provider", ConstString("SEL"), objc_flags);
+    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<false>, "SEL summary provider", ConstString("struct objc_selector"), objc_flags);
+    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<false>, "SEL summary provider", ConstString("objc_selector"), objc_flags);
+    AddCXXSummary(objc_category_sp, lldb_private::formatters::ObjCSELSummaryProvider<true>, "SEL summary provider", ConstString("objc_selector *"), objc_flags);
     
     AddScriptSummary(objc_category_sp, "lldb.formatters.objc.Class.Class_Summary", ConstString("Class"), objc_flags);
 #endif // LLDB_DISABLE_PYTHON
diff --git a/lldb/test/lang/cpp/char1632_t/Makefile b/lldb/test/lang/cpp/char1632_t/Makefile
new file mode 100644
index 00000000000..397bd743ea6
--- /dev/null
+++ b/lldb/test/lang/cpp/char1632_t/Makefile
@@ -0,0 +1,8 @@
+LEVEL = ../../../make
+
+CXX_SOURCES := main.cpp
+CFLAGS :=-arch x86_64 -gdwarf-2 -O0 -std=c++11
+
+clean: OBJECTS+=$(wildcard main.d.*)
+
+include $(LEVEL)/Makefile.rules
diff --git a/lldb/test/lang/cpp/char1632_t/TestChar1632T.py b/lldb/test/lang/cpp/char1632_t/TestChar1632T.py
new file mode 100644
index 00000000000..944c3272e85
--- /dev/null
+++ b/lldb/test/lang/cpp/char1632_t/TestChar1632T.py
@@ -0,0 +1,81 @@
+# coding=utf8
+"""
+Test that the C++11 support for char16_t and char32_t works correctly.
+"""
+
+import os, time
+import unittest2
+import lldb
+from lldbtest import *
+import lldbutil
+
+class Char1632TestCase(TestBase):
+
+    mydir = os.path.join("lang", "cpp", "char1632_t")
+
+    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
+    @dsym_test
+    def test_with_dsym(self):
+        """Test that the C++11 support for char16_t and char32_t works correctly."""
+        self.buildDsym()
+        self.char1632()
+
+    @dwarf_test
+    def test_with_dwarf(self):
+        """Test that the C++11 support for char16_t and char32_t works correctly."""
+        self.buildDwarf()
+        self.char1632()
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+        # Find the line number to break for main.cpp.
+        self.source = 'main.cpp'
+        self.line = line_number(self.source, '// Set break point at this line.')
+
+    def char1632(self):
+        """Test that the C++11 support for char16_t and char32_t works correctly."""
+        exe = os.path.join(os.getcwd(), "a.out")
+
+        # Create a target by the debugger.
+        target = self.dbg.CreateTarget(exe)
+        self.assertTrue(target, VALID_TARGET)
+
+        # Break on the struct declration statement in main.cpp.
+        lldbutil.run_break_set_by_file_and_line (self, "main.cpp", self.line)
+
+        # Now launch the process, and do not stop at entry point.
+        process = target.LaunchSimple(None, None, os.getcwd())
+
+        if not process:
+            self.fail("SBTarget.Launch() failed")
+
+        if self.TraceOn():
+             self.runCmd("frame variable")
+
+        # Check that we correctly report the const types
+        self.expect("frame variable cs16 cs32",
+            substrs = ['(const char16_t *) cs16 = ','(const char32_t *) cs32 = ','"hello world ྒྙྐ"','"hello world ྒྙྐ"'])
+
+        # Check that we correctly report the non-const types
+        self.expect("frame variable s16 s32",
+            substrs = ['(char16_t *) s16 = ','(char32_t *) s32 = ','"ﺸﺵۻ"','"ЕЙРГЖО"'])
+
+        self.runCmd("next") # step to after the string is nullified
+
+        # check that we don't crash on NULL
+        self.expect("frame variable s32",
+            substrs = ['(char32_t *) s32 = 0x00000000'])
+
+        self.runCmd("next")
+        self.runCmd("next")
+
+        # check that the new strings shows
+        self.expect("frame variable s16 s32",
+            substrs = ['(char16_t *) s16 = 0x','(char32_t *) s32 = ','"色ハ匂ヘト散リヌルヲ"','"෴"'])
+
+if __name__ == '__main__':
+    import atexit
+    lldb.SBDebugger.Initialize()
+    atexit.register(lambda: lldb.SBDebugger.Terminate())
+    unittest2.main()
diff --git a/lldb/test/lang/cpp/char1632_t/main.cpp b/lldb/test/lang/cpp/char1632_t/main.cpp
new file mode 100644
index 00000000000..fda951a7899
--- /dev/null
+++ b/lldb/test/lang/cpp/char1632_t/main.cpp
@@ -0,0 +1,21 @@
+//===-- main.c --------------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+
+int main (int argc, char const *argv[])
+{
+    auto cs16 = u"hello world ྒྙྐ";
+	auto cs32 = U"hello world ྒྙྐ";
+    char16_t *s16 = (char16_t *)u"ﺸﺵۻ";
+    char32_t *s32 = (char32_t *)U"ЕЙРГЖО";
+    s32 = nullptr; // Set break point at this line.
+    s32 = (char32_t *)U"෴";
+    s16 = (char16_t *)u"色ハ匂ヘト散リヌルヲ";
+    return 0;
+}