Port native Python-API to 3.x

With this change, liblldb is 95% of the way towards being able
to work under both Python 2.x and Python 3.x.  This should
introduce no functional change for Python 2.x, but for Python
3.x there are some important changes.  Primarily, these are:

1) PyString doesn't exist in Python 3.  Everything is a PyUnicode.
   To account for this, PythonString now stores a PyBytes instead
   of a PyString.  In Python 2, this is equivalent to a PyUnicode,
   and in Python 3, we do a conversion from PyUnicode to PyBytes
   and store the PyBytes.
2) PyInt doesn't exist in Python 3.  Everything is a PyLong.  To
   account for this, PythonInteger stores a PyLong instead of a
   PyInt.  In Python 2.x, this requires doing a conversion to
   PyLong when creating a PythonInteger from a PyInt.  In 3.x,
   there is no PyInt anyway, so we can assume everything is a
   PyLong.
3) PyFile_FromFile doesn't exist in Python 3.  Instead there is a
   PyFile_FromFd.  This is not addressed in this patch because it
   will require quite a large change to plumb fd's all the way
   through the system into the ScriptInterpreter.  This is the only
   remaining piece of the puzzle to get LLDB supporting Python 3.x.

Being able to run the test suite is not addressed in this patch.
After the extension module can compile and you can enter an embedded
3.x interpreter, the test suite will be addressed in a followup.

llvm-svn: 249886

3 files changed, 227 insertions, 99 deletions

diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
index 4f81d07b3e5..5e94afc0a12 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
@@ -71,10 +71,18 @@ PythonObject::GetObjectType() const
         return PyObjectType::List;
     if (PyDict_Check(m_py_obj))
         return PyObjectType::Dictionary;
+    if (PyUnicode_Check(m_py_obj))
+        return PyObjectType::String;
+    if (PyLong_Check(m_py_obj))
+        return PyObjectType::Integer;
+#if PY_MAJOR_VERSION < 3
+    // These functions don't exist in Python 3.x.  PyString is PyUnicode
+    // and PyInt is PyLong.
     if (PyString_Check(m_py_obj))
         return PyObjectType::String;
-    if (PyInt_Check(m_py_obj) || PyLong_Check(m_py_obj))
+    if (PyInt_Check(m_py_obj))
         return PyObjectType::Integer;
+#endif
     return PyObjectType::Unknown;
 }
 
@@ -142,14 +150,16 @@ PythonString::PythonString (const PythonObject &object) :
     Reset(object.get()); // Use "Reset()" to ensure that py_obj is a string
 }
 
-PythonString::PythonString (llvm::StringRef string) :
-    PythonObject(PyString_FromStringAndSize(string.data(), string.size()))
+PythonString::PythonString(llvm::StringRef string)
+    : PythonObject()
 {
+    SetString(string);
 }
 
-PythonString::PythonString(const char *string) :
-    PythonObject(PyString_FromString(string))
+PythonString::PythonString(const char *string)
+    : PythonObject()
 {
+    SetString(llvm::StringRef(string));
 }
 
 PythonString::PythonString () :
@@ -162,20 +172,53 @@ PythonString::~PythonString ()
 }
 
 bool
-PythonString::Reset (PyObject *py_obj)
+PythonString::Check(PyObject *py_obj)
 {
-    if (py_obj && PyString_Check(py_obj))
-        return PythonObject::Reset(py_obj);
-    
-    PythonObject::Reset(nullptr);
-    return py_obj == nullptr;
+    if (!py_obj)
+        return false;
+#if PY_MAJOR_VERSION >= 3
+    // Python 3 does not have PyString objects, only PyUnicode.
+    return PyUnicode_Check(py_obj);
+#else
+    return PyUnicode_Check(py_obj) || PyString_Check(py_obj);
+#endif
+}
+
+bool
+PythonString::Reset(PyObject *py_obj)
+{
+    if (!PythonString::Check(py_obj))
+    {
+        PythonObject::Reset(nullptr);
+        return false;
+    }
+
+// Convert this to a PyBytes object, and only store the PyBytes.  Note that in
+// Python 2.x, PyString and PyUnicode are interchangeable, and PyBytes is an alias
+// of PyString.  So on 2.x, if we get into this branch, we already have a PyBytes.
+    //#if PY_MAJOR_VERSION >= 3
+    if (PyUnicode_Check(py_obj))
+    {
+        PyObject *unicode = py_obj;
+        py_obj = PyUnicode_AsUTF8String(py_obj);
+        Py_XDECREF(unicode);
+    }
+    //#endif
+
+    assert(PyBytes_Check(py_obj) && "PythonString::Reset received a non-string");
+    return PythonObject::Reset(py_obj);
 }
 
 llvm::StringRef
 PythonString::GetString() const
 {
     if (m_py_obj)
-        return llvm::StringRef(PyString_AsString(m_py_obj), GetSize());
+    {
+        Py_ssize_t size;
+        char *c;
+        PyBytes_AsStringAndSize(m_py_obj, &c, &size);
+        return llvm::StringRef(c, size);
+    }
     return llvm::StringRef();
 }
 
@@ -183,14 +226,21 @@ size_t
 PythonString::GetSize() const
 {
     if (m_py_obj)
-        return PyString_Size(m_py_obj);
+        return PyBytes_Size(m_py_obj);
     return 0;
 }
 
 void
 PythonString::SetString (llvm::StringRef string)
 {
+#if PY_MAJOR_VERSION >= 3
+    PyObject *unicode = PyUnicode_FromStringAndSize(string.data(), string.size());
+    PyObject *bytes = PyUnicode_AsUTF8String(unicode);
+    PythonObject::Reset(bytes);
+    Py_XDECREF(unicode);
+#else
     PythonObject::Reset(PyString_FromStringAndSize(string.data(), string.size()));
+#endif
 }
 
 StructuredData::StringSP
@@ -229,16 +279,44 @@ PythonInteger::~PythonInteger ()
 }
 
 bool
-PythonInteger::Reset (PyObject *py_obj)
+PythonInteger::Check(PyObject *py_obj)
+{
+    if (!py_obj)
+        return false;
+
+#if PY_MAJOR_VERSION >= 3
+    // Python 3 does not have PyInt_Check.  There is only one type of
+    // integral value, long.
+    return PyLong_Check(py_obj);
+#else
+    return PyLong_Check(py_obj) || PyInt_Check(py_obj);
+#endif
+}
+
+bool
+PythonInteger::Reset(PyObject *py_obj)
 {
-    if (py_obj)
+    if (!PythonInteger::Check(py_obj))
     {
-        if (PyInt_Check (py_obj) || PyLong_Check(py_obj))
-            return PythonObject::Reset(py_obj);
+        PythonObject::Reset(nullptr);
+        return false;
     }
-    
-    PythonObject::Reset(nullptr);
-    return py_obj == nullptr;
+
+#if PY_MAJOR_VERSION < 3
+    // Always store this as a PyLong, which makes interoperability between
+    // Python 2.x and Python 3.x easier.  This is only necessary in 2.x,
+    // since 3.x doesn't even have a PyInt.
+    if (PyInt_Check(py_obj))
+    {
+        PyObject *py_long = PyLong_FromLongLong(PyInt_AsLong(py_obj));
+        Py_XDECREF(py_obj);
+        py_obj = py_long;
+    }
+#endif
+
+    assert(PyLong_Check(py_obj) && "Couldn't get a PyLong from this PyObject");
+
+    return PythonObject::Reset(py_obj);
 }
 
 int64_t
@@ -246,10 +324,9 @@ PythonInteger::GetInteger() const
 {
     if (m_py_obj)
     {
-        if (PyInt_Check(m_py_obj))
-            return PyInt_AsLong(m_py_obj);
-        else if (PyLong_Check(m_py_obj))
-            return PyLong_AsLongLong(m_py_obj);
+        assert(PyLong_Check(m_py_obj) && "PythonInteger::GetInteger has a PyObject that isn't a PyLong");
+
+        return PyLong_AsLongLong(m_py_obj);
     }
     return UINT64_MAX;
 }
@@ -272,13 +349,8 @@ PythonInteger::CreateStructuredInteger() const
 // PythonList
 //----------------------------------------------------------------------
 
-PythonList::PythonList (bool create_empty) :
-    PythonObject(create_empty ? PyList_New(0) : nullptr)
-{
-}
-
-PythonList::PythonList (uint32_t count) :
-    PythonObject(PyList_New(count))
+PythonList::PythonList()
+    : PythonObject(PyList_New(0))
 {
 }
 
@@ -300,13 +372,23 @@ PythonList::~PythonList ()
 }
 
 bool
-PythonList::Reset (PyObject *py_obj)
+PythonList::Check(PyObject *py_obj)
 {
-    if (py_obj && PyList_Check(py_obj))
-        return PythonObject::Reset(py_obj);
-    
-    PythonObject::Reset(nullptr);
-    return py_obj == nullptr;
+    if (!py_obj)
+        return false;
+    return PyList_Check(py_obj);
+}
+
+bool
+PythonList::Reset(PyObject *py_obj)
+{
+    if (!PythonList::Check(py_obj))
+    {
+        PythonObject::Reset(nullptr);
+        return false;
+    }
+
+    return PythonObject::Reset(py_obj);
 }
 
 uint32_t
@@ -356,8 +438,8 @@ PythonList::CreateStructuredArray() const
 // PythonDictionary
 //----------------------------------------------------------------------
 
-PythonDictionary::PythonDictionary (bool create_empty) :
-PythonObject(create_empty ? PyDict_New() : nullptr)
+PythonDictionary::PythonDictionary()
+    : PythonObject(PyDict_New())
 {
 }
 
@@ -379,13 +461,24 @@ PythonDictionary::~PythonDictionary ()
 }
 
 bool
-PythonDictionary::Reset (PyObject *py_obj)
+PythonDictionary::Check(PyObject *py_obj)
 {
-    if (py_obj && PyDict_Check(py_obj))
-        return PythonObject::Reset(py_obj);
-    
-    PythonObject::Reset(nullptr);
-    return py_obj == nullptr;
+    if (!py_obj)
+        return false;
+
+    return PyDict_Check(py_obj);
+}
+
+bool
+PythonDictionary::Reset(PyObject *py_obj)
+{
+    if (!PythonDictionary::Check(py_obj))
+    {
+        PythonObject::Reset(nullptr);
+        return false;
+    }
+
+    return PythonObject::Reset(py_obj);
 }
 
 uint32_t
@@ -423,8 +516,11 @@ PythonDictionary::GetItemForKeyAsString (const PythonString &key, const char *fa
     if (m_py_obj && key)
     {
         PyObject *py_obj = PyDict_GetItem(m_py_obj, key.get());
-        if (py_obj && PyString_Check(py_obj))
-            return PyString_AsString(py_obj);
+        if (py_obj && PythonString::Check(py_obj))
+        {
+            PythonString str(py_obj);
+            return str.GetString().data();
+        }
     }
     return fail_value;
 }
@@ -435,13 +531,10 @@ PythonDictionary::GetItemForKeyAsInteger (const PythonString &key, int64_t fail_
     if (m_py_obj && key)
     {
         PyObject *py_obj = PyDict_GetItem(m_py_obj, key.get());
-        if (py_obj)
+        if (PythonInteger::Check(py_obj))
         {
-            if (PyInt_Check(py_obj))
-                return PyInt_AsLong(py_obj);
-
-            if (PyLong_Check(py_obj))
-                return PyLong_AsLong(py_obj);
+            PythonInteger int_obj(py_obj);
+            return int_obj.GetInteger();
         }
     }
     return fail_value;
@@ -452,7 +545,7 @@ PythonDictionary::GetKeys () const
 {
     if (m_py_obj)
         return PythonList(PyDict_Keys(m_py_obj));
-    return PythonList(true);
+    return PythonList();
 }
 
 PythonString
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
index 43963589cc1..33275d55468 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
@@ -156,7 +156,7 @@ enum class PyObjectType
     protected:
         PyObject* m_py_obj;
     };
-    
+
     class PythonString: public PythonObject
     {
     public:
@@ -167,6 +167,8 @@ enum class PyObjectType
         PythonString (const char *string);
         virtual ~PythonString ();
 
+        static bool Check(PyObject *py_obj);
+
         virtual bool
         Reset (PyObject* py_obj = NULL);
 
@@ -180,7 +182,7 @@ enum class PyObjectType
 
         StructuredData::StringSP CreateStructuredString() const;
     };
-    
+
     class PythonInteger: public PythonObject
     {
     public:
@@ -190,7 +192,9 @@ enum class PyObjectType
         PythonInteger (const PythonObject &object);
         PythonInteger (int64_t value);
         virtual ~PythonInteger ();
-        
+
+        static bool Check(PyObject *py_obj);
+
         virtual bool
         Reset (PyObject* py_obj = NULL);
 
@@ -205,13 +209,13 @@ enum class PyObjectType
     class PythonList: public PythonObject
     {
     public:
-        
-        PythonList (bool create_empty);
+      PythonList();
         PythonList (PyObject* py_obj);
         PythonList (const PythonObject &object);
-        PythonList (uint32_t count);
         virtual ~PythonList ();
-        
+
+        static bool Check(PyObject *py_obj);
+
         virtual bool
         Reset (PyObject* py_obj = NULL);
 
@@ -231,12 +235,13 @@ enum class PyObjectType
     class PythonDictionary: public PythonObject
     {
     public:
-        
-        explicit PythonDictionary (bool create_empty);
+      PythonDictionary();
         PythonDictionary (PyObject* object);
         PythonDictionary (const PythonObject &object);
         virtual ~PythonDictionary ();
-        
+
+        static bool Check(PyObject *py_obj);
+
         virtual bool
         Reset (PyObject* object = NULL);
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index b7e6cb1f44e..ea93cf04fd1 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -46,6 +46,7 @@
 #endif
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/STLExtras.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -79,6 +80,29 @@ static ScriptInterpreterPython::SWIGPythonCallThreadPlan g_swig_call_thread_plan
 
 static bool g_initialized = false;
 
+#if PY_MAJOR_VERSION >= 3 && defined(LLDB_PYTHON_HOME)
+typedef wchar_t PythonHomeChar;
+#else
+typedef char PythonHomeChar;
+#endif
+
+PythonHomeChar *
+GetDesiredPythonHome()
+{
+#if defined(LLDB_PYTHON_HOME)
+#if PY_MAJOR_VERSION >= 3
+    size_t size = 0;
+    static PythonHomeChar *g_python_home = Py_DecodeLocale(LLDB_PYTHON_HOME, &size);
+    return g_python_home;
+#else
+    static PythonHomeChar *g_python_home = LLDB_PYTHON_HOME;
+    return g_python_home;
+#endif
+#else
+    return nullptr;
+#endif
+}
+
 static std::string
 ReadPythonBacktrace (PyObject* py_backtrace);
 
@@ -116,7 +140,7 @@ ScriptInterpreterPython::Locker::DoAcquireLock()
     // place outside of Python (e.g. printing to screen, waiting for the network, ...)
     // in that case, _PyThreadState_Current will be NULL - and we would be unable
     // to set the asynchronous exception - not a desirable situation
-    m_python_interpreter->SetThreadState (_PyThreadState_Current);
+    m_python_interpreter->SetThreadState(PyThreadState_Get());
     m_python_interpreter->IncrementLockCount();
     return true;
 }
@@ -908,13 +932,13 @@ ScriptInterpreterPython::Interrupt()
 
     if (IsExecutingPython())
     {
-        PyThreadState* state = _PyThreadState_Current;
+        PyThreadState *state = PyThreadState_Get();
         if (!state)
             state = GetThreadState();
         if (state)
         {
             long tid = state->thread_id;
-            _PyThreadState_Current = state;
+            PyThreadState_Swap(state);
             int num_threads = PyThreadState_SetAsyncExc(tid, PyExc_KeyboardInterrupt);
             if (log)
                 log->Printf("ScriptInterpreterPython::Interrupt() sending PyExc_KeyboardInterrupt (tid = %li, num_threads = %i)...", tid, num_threads);
@@ -1124,15 +1148,16 @@ ScriptInterpreterPython::ExecuteMultipleLines (const char *in_string, const Exec
     
     if (in_string != nullptr)
     {
-        struct _node *compiled_node = PyParser_SimpleParseString (in_string, Py_file_input);
+#if PY_MAJOR_VERSION >= 3
+        PyObject *code_object = Py_CompileString(in_string, "temp.py", Py_file_input);
+#else
+        PyCodeObject *code_object = nullptr;
+        struct _node *compiled_node = PyParser_SimpleParseString(in_string, Py_file_input);
         if (compiled_node)
-        {
-            PyCodeObject *compiled_code = PyNode_Compile (compiled_node, "temp.py");
-            if (compiled_code)
-            {
-              return_value.Reset(PyEval_EvalCode (compiled_code, globals.get(), locals.get()));
-            }
-        }
+            code_object = PyNode_Compile(compiled_node, "temp.py");
+#endif
+        if (code_object)
+            return_value.Reset(PyEval_EvalCode(code_object, globals.get(), locals.get()));
     }
 
     py_error = PyErr_Occurred ();
@@ -1152,7 +1177,11 @@ ScriptInterpreterPython::ExecuteMultipleLines (const char *in_string, const Exec
         std::string bt = ReadPythonBacktrace(traceback);
         
         if (value && value != Py_None)
-            error.SetErrorStringWithFormat("%s\n%s", PyString_AsString(PyObject_Str(value)),bt.c_str());
+        {
+            PythonString str(value);
+            llvm::StringRef value_str(str.GetString());
+            error.SetErrorStringWithFormat("%s\n%s", value_str.str().c_str(), bt.c_str());
+        }
         else
             error.SetErrorStringWithFormat("%s",bt.c_str());
         Py_XDECREF(type);
@@ -2342,8 +2371,12 @@ ReadPythonBacktrace (PyObject* py_backtrace)
                         if (stringIO_getvalue && stringIO_getvalue != Py_None)
                         {
                             printTB_string = PyObject_CallObject (stringIO_getvalue,nullptr);
-                            if (printTB_string && printTB_string != Py_None && PyString_Check(printTB_string))
-                                retval.assign(PyString_AsString(printTB_string));
+                            if (printTB_string && PythonString::Check(printTB_string))
+                            {
+                                PythonString str(printTB_string);
+                                llvm::StringRef string_data(str.GetString());
+                                retval.assign(string_data.data(), string_data.size());
+                            }
                         }
                     }
                 }
@@ -2924,14 +2957,13 @@ ScriptInterpreterPython::GetShortHelpForCommandObject (StructuredData::GenericSP
         PyErr_Print();
         PyErr_Clear();
     }
-    
-    if (py_return != nullptr && py_return != Py_None)
+
+    if (py_return != Py_None && PythonString::Check(py_return))
     {
-        if (PyString_Check(py_return))
-        {
-            dest.assign(PyString_AsString(py_return));
-            got_string = true;
-        }
+        PythonString py_string(py_return);
+        llvm::StringRef return_data(py_string.GetString());
+        dest.assign(return_data.data(), return_data.size());
+        got_string = true;
     }
     Py_XDECREF(py_return);
     
@@ -2997,13 +3029,11 @@ ScriptInterpreterPython::GetFlagsForCommandObject (StructuredData::GenericSP cmd
         PyErr_Print();
         PyErr_Clear();
     }
-    
-    if (py_return != nullptr && py_return != Py_None)
+
+    if (py_return != Py_None && PythonInteger::Check(py_return))
     {
-        if (PyInt_Check(py_return))
-            result = (uint32_t)PyInt_AsLong(py_return);
-        else if (PyLong_Check(py_return))
-            result = (uint32_t)PyLong_AsLong(py_return);
+        PythonInteger int_value(py_return);
+        result = int_value.GetInteger();
     }
     Py_XDECREF(py_return);
     
@@ -3071,14 +3101,13 @@ ScriptInterpreterPython::GetLongHelpForCommandObject (StructuredData::GenericSP
         PyErr_Print();
         PyErr_Clear();
     }
-    
-    if (py_return != nullptr && py_return != Py_None)
+
+    if (py_return != Py_None && PythonString::Check(py_return))
     {
-        if (PyString_Check(py_return))
-        {
-            dest.assign(PyString_AsString(py_return));
-            got_string = true;
-        }
+        PythonString str(py_return);
+        llvm::StringRef str_data(str.GetString());
+        dest.assign(str_data.data(), str_data.size());
+        got_string = true;
     }
     Py_XDECREF(py_return);
     
@@ -3164,8 +3193,9 @@ ScriptInterpreterPython::InitializePrivate ()
     stdin_tty_state.Save(STDIN_FILENO, false);
 
 #if defined(LLDB_PYTHON_HOME)
-    Py_SetPythonHome(LLDB_PYTHON_HOME);
+    Py_SetPythonHome(GetDesiredPythonHome());
 #endif
+
     PyGILState_STATE gstate;
     Log *log (lldb_private::GetLogIfAllCategoriesSet (LIBLLDB_LOG_SCRIPT | LIBLLDB_LOG_VERBOSE));
     bool threads_already_initialized = false;