Fix some issues with bytes and strings in Python 3.

SBProcess::ReadMemory and other related functions such as
WriteMemory are returning Python string() objects.  This means
that in Python 3 that are returning Unicode objects.  In reality
they should be returning bytes objects which is the same as a string
in Python 2, but different in Python 3.  This patch updates the
generated SWIG code to return Python bytes objects for all
memory related functions.

One quirk of this patch is that the C++ signature of ReadCStringFromMemory
has it writing c-string data into a void*.  This confuses our swig
typemaps which expect that a void* means byte data.  So I hacked up
a custom typemap which maps this specific function to treat the
void* as string data instead of byte data.

llvm-svn: 258743

4 files changed, 15 insertions, 14 deletions

diff --git a/lldb/packages/Python/lldbsuite/test/lldbutil.py b/lldb/packages/Python/lldbsuite/test/lldbutil.py
index 7aeffa37671..ea34ed6271e 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbutil.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbutil.py
@@ -84,7 +84,7 @@ def int_to_bytearray(val, bytesize):
         return None
 
     packed = struct.pack(fmt, val)
-    return bytearray(list(map(ord, packed)))
+    return bytearray(packed)
 
 def bytearray_to_int(bytes, bytesize):
     """Utility function to convert a bytearray into an integer.
@@ -108,7 +108,7 @@ def bytearray_to_int(bytes, bytesize):
     else:
         return None
 
-    unpacked = struct.unpack(fmt, str(bytes))
+    unpacked = struct.unpack_from(fmt, bytes)
     return unpacked[0]
 
 
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/process/TestProcessAPI.py b/lldb/packages/Python/lldbsuite/test/python_api/process/TestProcessAPI.py
index f312bc8a924..31d129b67ee 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/process/TestProcessAPI.py
+++ b/lldb/packages/Python/lldbsuite/test/python_api/process/TestProcessAPI.py
@@ -56,7 +56,7 @@ class ProcessAPITestCase(TestBase):
 
         self.expect(content, "Result from SBProcess.ReadMemory() matches our expected output: 'x'",
                     exe=False,
-            startstr = 'x')
+            startstr = b'x')
 
         # Read (char *)my_char_ptr.
         val = frame.FindValue("my_char_ptr", lldb.eValueTypeVariableGlobal)
@@ -154,7 +154,7 @@ class ProcessAPITestCase(TestBase):
 
         self.expect(content, "Result from SBProcess.ReadMemory() matches our expected output: 'a'",
                     exe=False,
-            startstr = 'a')
+            startstr = b'a')
 
     @add_test_categories(['pyapi'])
     def test_access_my_int(self):
@@ -206,9 +206,8 @@ class ProcessAPITestCase(TestBase):
         # But we want to use the WriteMemory() API to assign 256 to the variable.
 
         # Now use WriteMemory() API to write 256 into the global variable.
-        new_value = str(bytes)
         error = lldb.SBError()
-        result = process.WriteMemory(location, new_value, error)
+        result = process.WriteMemory(location, bytes, error)
         if not error.Success() or result != byteSize:
             self.fail("SBProcess.WriteMemory() failed")
 
@@ -230,14 +229,11 @@ class ProcessAPITestCase(TestBase):
         if not error.Success():
             self.fail("SBProcess.ReadMemory() failed")
 
-        # Use "ascii" as the encoding because each element of 'content' is in the range [0..255].
-        new_bytes = bytearray(content, "ascii")
-
         # The bytearray_to_int utility function expects a little endian bytearray.
         if byteOrder == lldb.eByteOrderBig:
             new_bytes.reverse()
 
-        new_value = bytearray_to_int(new_bytes, byteSize)
+        new_value = bytearray_to_int(content, byteSize)
         if new_value != 256:
             self.fail("Memory content read from 'my_int' does not match (int)256")
 
diff --git a/lldb/scripts/Python/python-typemaps.swig b/lldb/scripts/Python/python-typemaps.swig
index 659199f1fd0..82e33e1bc60 100644
--- a/lldb/scripts/Python/python-typemaps.swig
+++ b/lldb/scripts/Python/python-typemaps.swig
@@ -93,6 +93,9 @@
    }
    $1 = (char *) malloc($2);
 }
+// SBProcess::ReadCStringFromMemory() uses a void*, but needs to be treated
+// as char data instead of byte data.
+%typemap(in) (void *char_buf, size_t size) = (char *dst, size_t dst_len);
 
 // Return the char buffer.  Discarding any previous return result
 // See also SBThread::GetStopDescription.
@@ -108,6 +111,9 @@
    }
    free($1);
 }
+// SBProcess::ReadCStringFromMemory() uses a void*, but needs to be treated
+// as char data instead of byte data.
+%typemap(argout) (void *char_buf, size_t size) = (char *dst, size_t dst_len);
 
 
 // typemap for an outgoing buffer
@@ -231,9 +237,8 @@
       $result = Py_None;
       Py_INCREF($result);
    } else {
-      llvm::StringRef ref(static_cast<const char*>($1), result);
-      lldb_private::PythonString string(ref);
-      $result = string.release();
+      lldb_private::PythonBytes bytes(static_cast<const uint8_t*>($1), result);
+      $result = bytes.release();
    }
    free($1);
 }
diff --git a/lldb/scripts/interface/SBProcess.i b/lldb/scripts/interface/SBProcess.i
index 1571ebc4cb6..0ce45f7d7be 100644
--- a/lldb/scripts/interface/SBProcess.i
+++ b/lldb/scripts/interface/SBProcess.i
@@ -296,7 +296,7 @@ public:
     ") ReadCStringFromMemory;
 
     size_t
-    ReadCStringFromMemory (addr_t addr, void *buf, size_t size, lldb::SBError &error);
+    ReadCStringFromMemory (addr_t addr, void *char_buf, size_t size, lldb::SBError &error);
 
     %feature("autodoc", "
     Reads an unsigned integer from memory given a byte size and an address.