Making linking against Python simpler on Windows.

This patch deprecates the three Python CMake variables in favor of
a single variable PYTHON_HOME which points to the root of a python
installation.  Since building Python doesn't output the files in
a structure that is compatible with the PYTHONHOME environment
variable, we also provide a script install_custom_python.py which
will copy the output of a custom python build to the correct
directory structure.

The supported workflow after this patch will be to build python
once for each configuration and architecture {Debug,Release} x {x86,x64}
and then run the script.  Then run CMake specifying -DPYTHON_HOME=<path>

The first time you do this will probably require you to delete your
CMake cache.

The old workflow is still supported during a transitionary period,
but a warning is printed at CMake time, and this will eventually
be removed.

Differential Revision: http://reviews.llvm.org/D8979

llvm-svn: 234660

4 files changed, 190 insertions, 68 deletions

diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake
index e01ae95157d..664113079df 100644
--- a/lldb/cmake/modules/LLDBConfig.cmake
+++ b/lldb/cmake/modules/LLDBConfig.cmake
@@ -1,3 +1,7 @@
+set(LLDB_PROJECT_ROOT ${CMAKE_CURRENT_SOURCE_DIR})

+set(LLDB_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/source")

+set(LLDB_INCLUDE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/include")

+

 if ( CMAKE_SYSTEM_NAME MATCHES "Windows" )

   set(LLDB_DEFAULT_DISABLE_PYTHON 0)

   set(LLDB_DEFAULT_DISABLE_CURSES 1)

@@ -10,25 +14,16 @@ else()
     set(LLDB_DEFAULT_DISABLE_CURSES 0)

   endif()

 endif()

+

 set(LLDB_DISABLE_PYTHON ${LLDB_DEFAULT_DISABLE_PYTHON} CACHE BOOL

   "Disables the Python scripting integration.")

 set(LLDB_DISABLE_CURSES ${LLDB_DEFAULT_DISABLE_CURSES} CACHE BOOL

   "Disables the Curses integration.")

 

-if ( LLDB_DISABLE_PYTHON )

-  set(LLDB_DEFAULT_ENABLE_PYTHON_SCRIPTS_SWIG_API_GENERATION 0)

-else ()

-  set(LLDB_DEFAULT_ENABLE_PYTHON_SCRIPTS_SWIG_API_GENERATION 1)

-endif ()

-

-set(LLDB_ENABLE_PYTHON_SCRIPTS_SWIG_API_GENERATION ${LLDB_DEFAULT_ENABLE_PYTHON_SCRIPTS_SWIG_API_GENERATION} CACHE BOOL

+set(LLDB_ENABLE_PYTHON_SCRIPTS_SWIG_API_GENERATION 1 CACHE BOOL

   "Enables using new Python scripts for SWIG API generation .")  

-

-set(LLDB_PROJECT_ROOT ${CMAKE_CURRENT_SOURCE_DIR})

-set(LLDB_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/source")

-set(LLDB_INCLUDE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/include")

-

-set(LLDB_DISABLE_PYTHON 0 CACHE BOOL "Disables the Python scripting integration.")

+set(LLDB_RELOCATABLE_PYTHON 0 CACHE BOOL

+  "Causes LLDB to use the PYTHONHOME environment variable to locate Python.")

 

 if ((NOT MSVC) OR MSVC12)

   add_definitions( -DHAVE_ROUND )

@@ -45,15 +40,39 @@ if (NOT LLDB_DISABLE_PYTHON)
       set(CMAKE_LIBRARY_ARCHITECTURE "x86_64-linux-gnu")

     endif()

   endif()

-  if (MSVC)

-    if ("${PYTHON_INCLUDE_DIR}" STREQUAL "" OR "${PYTHON_LIBRARY}" STREQUAL "")

-      message("-- LLDB Embedded python disabled.  Embedding python on Windows requires "

-              "manually specifying PYTHON_INCLUDE_DIR *and* PYTHON_LIBRARY")

-      set(LLDB_DISABLE_PYTHON 1)

+  

+  if ("${CMAKE_SYSTEM_NAME}" STREQUAL "Windows")

+    if (NOT "${PYTHON_HOME}" STREQUAL "")

+      file(TO_CMAKE_PATH "${PYTHON_HOME}" PYTHON_HOME)

+      if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")

+        file(TO_CMAKE_PATH "${PYTHON_HOME}/python_d.exe" PYTHON_EXECUTABLE)

+        file(TO_CMAKE_PATH "${PYTHON_HOME}/libs/python27_d.lib" PYTHON_LIBRARY)

+        file(TO_CMAKE_PATH "${PYTHON_HOME}/python27_d.dll" PYTHON_DLL)

+      else()

+        file(TO_CMAKE_PATH "${PYTHON_HOME}/python.exe" PYTHON_EXECUTABLE)

+        file(TO_CMAKE_PATH "${PYTHON_HOME}/libs/python27.lib" PYTHON_LIBRARY)

+        file(TO_CMAKE_PATH "${PYTHON_HOME}/python27.dll" PYTHON_DLL)

+      endif()

+      

+      file(TO_CMAKE_PATH "${PYTHON_HOME}/Include" PYTHON_INCLUDE_DIR)

+      if (NOT LLDB_RELOCATABLE_PYTHON)

+        add_definitions( -DLLDB_PYTHON_HOME="${PYTHON_HOME}" )

+      endif()

     else()

+      message("Embedding Python on Windows without specifying a value for PYTHON_HOME is deprecated.  Support for this will be dropped soon.")

+        

+      if ("${PYTHON_INCLUDE_DIR}" STREQUAL "" OR "${PYTHON_LIBRARY}" STREQUAL "")

+        message("-- LLDB Embedded python disabled.  Embedding python on Windows requires "

+                "manually specifying PYTHON_INCLUDE_DIR *and* PYTHON_LIBRARY")

+        set(LLDB_DISABLE_PYTHON 1)

+      endif()

+    endif()

+    

+    if (PYTHON_LIBRARY)

       message("-- Found PythonLibs: ${PYTHON_LIBRARY}")

       include_directories(${PYTHON_INCLUDE_DIR})

     endif()

+    

   else()

     find_package(PythonLibs REQUIRED)

     include_directories(${PYTHON_INCLUDE_DIRS})

diff --git a/lldb/scripts/install_custom_python.py b/lldb/scripts/install_custom_python.py
new file mode 100644
index 00000000000..5a8f48aac94
--- /dev/null
+++ b/lldb/scripts/install_custom_python.py
@@ -0,0 +1,134 @@
+""" Copies the build output of a custom python interpreter to a directory
+    structure that mirrors that of an official Python distribution.
+
+    --------------------------------------------------------------------------
+    File:           install_custom_python.py
+
+    Overview:       Most users build LLDB by linking against the standard
+                    Python distribution installed on their system.  Occasionally
+                    a user may want to build their own version of Python, and on
+                    platforms such as Windows this is a hard requirement.  This
+                    script will take the build output of a custom interpreter and
+                    install it into a canonical structure that mirrors that of an
+                    official Python distribution, thus allowing PYTHONHOME to be
+                    set appropriately.
+
+    Gotchas:        None.
+
+    Copyright:      None.
+    --------------------------------------------------------------------------
+
+"""
+
+import argparse
+import itertools
+import os
+import shutil
+import sys
+
+def copy_one_file(dest_dir, source_dir, filename):
+    source_path = os.path.join(source_dir, filename)
+    dest_path = os.path.join(dest_dir, filename)
+    print 'Copying file %s ==> %s...' % (source_path, dest_path)
+    shutil.copyfile(source_path, dest_path)
+
+def copy_named_files(dest_dir, source_dir, files, extensions, copy_debug_suffix_also):
+    for (file, ext) in itertools.product(files, extensions):
+        copy_one_file(dest_dir, source_dir, file + '.' + ext)
+        if copy_debug_suffix_also:
+            copy_one_file(dest_dir, source_dir, file + '_d.' + ext)
+
+def copy_subdirectory(dest_dir, source_dir, subdir):
+    dest_dir = os.path.join(dest_dir, subdir)
+    source_dir = os.path.join(source_dir, subdir)
+    print 'Copying directory %s ==> %s...' % (source_dir, dest_dir)
+    shutil.copytree(source_dir, dest_dir)
+
+def copy_distro(dest_dir, dest_subdir, source_dir, source_prefix):
+    dest_dir = os.path.join(dest_dir, dest_subdir)
+
+    print 'Copying distribution %s ==> %s' % (source_dir, dest_dir)
+
+    os.mkdir(dest_dir)
+    PCbuild_dir = os.path.join(source_dir, 'PCbuild')
+    if source_prefix:
+        PCbuild_dir = os.path.join(PCbuild_dir, source_prefix)
+    # First copy the files that go into the root of the new distribution. This
+    # includes the Python executables, python27(_d).dll, and relevant PDB files.
+    print 'Copying Python executables...'
+    copy_named_files(dest_dir, PCbuild_dir, ['w9xpopen'], ['exe', 'pdb'], False)
+    copy_named_files(dest_dir, PCbuild_dir, ['python_d', 'pythonw_d'], ['exe'], False)
+    copy_named_files(dest_dir, PCbuild_dir, ['python', 'pythonw'], ['exe', 'pdb'], False)
+    copy_named_files(dest_dir, PCbuild_dir, ['python27'], ['dll', 'pdb'], True)
+
+    # Next copy everything in the Include directory.
+    print 'Copying Python include directory'
+    copy_subdirectory(dest_dir, source_dir, 'Include')
+
+    # Copy Lib folder (builtin Python modules)
+    print 'Copying Python Lib directory'
+    copy_subdirectory(dest_dir, source_dir, 'Lib')
+
+    # Copy tools folder.  These are probably not necessary, but we copy them anyway to
+    # match an official distribution as closely as possible.  Note that we don't just copy
+    # the subdirectory recursively.  The source distribution ships with many more tools
+    # than what you get by installing python regularly.  We only copy the tools that appear
+    # in an installed distribution.
+    tools_dest_dir = os.path.join(dest_dir, 'Tools')
+    tools_source_dir = os.path.join(source_dir, 'Tools')
+    os.mkdir(tools_dest_dir)
+    copy_subdirectory(tools_dest_dir, tools_source_dir, 'i18n')
+    copy_subdirectory(tools_dest_dir, tools_source_dir, 'pynche')
+    copy_subdirectory(tools_dest_dir, tools_source_dir, 'scripts')
+    copy_subdirectory(tools_dest_dir, tools_source_dir, 'versioncheck')
+    copy_subdirectory(tools_dest_dir, tools_source_dir, 'webchecker')
+
+    pyd_names = ['_ctypes', '_ctypes_test', '_elementtree', '_multiprocessing', '_socket',
+                 '_testcapi', 'pyexpat', 'select', 'unicodedata', 'winsound']
+
+    # Copy builtin extension modules (pyd files)
+    dlls_dir = os.path.join(dest_dir, 'DLLs')
+    os.mkdir(dlls_dir)
+    print 'Copying DLLs directory'
+    copy_named_files(dlls_dir, PCbuild_dir, pyd_names, ['pyd', 'pdb'], True)
+
+    # Copy libs folder (implibs for the pyd files)
+    libs_dir = os.path.join(dest_dir, 'libs')
+    os.mkdir(libs_dir)
+    print 'Copying libs directory'
+    copy_named_files(libs_dir, PCbuild_dir, pyd_names, ['lib'], False)
+    copy_named_files(libs_dir, PCbuild_dir, ['python27'], ['lib'], True)
+
+
+parser = argparse.ArgumentParser(description='Install a custom Python distribution')
+parser.add_argument('--source', required=True, help='The root of the source tree where Python is built.')
+parser.add_argument('--dest', required=True, help='The location to install the Python distributions.')
+parser.add_argument('--overwrite', default=False, action='store_true', help='If the destination directory already exists, destroys its contents first.')
+parser.add_argument('--silent', default=False, action='store_true', help='If --overwite was specified, suppress confirmation before deleting a directory tree.')
+
+args = parser.parse_args()
+
+args.source = os.path.normpath(args.source)
+args.dest = os.path.normpath(args.dest)
+
+if not os.path.exists(args.source):
+    print 'The source directory %s does not exist.  Exiting...'
+    sys.exit(1)
+
+if os.path.exists(args.dest):
+    if not args.overwrite:
+        print 'The destination directory \'%s\' already exists and --overwrite was not specified.  Exiting...' % args.dest
+        sys.exit(1)
+    while not args.silent:
+        print 'Ok to recursively delete \'%s\' and all contents (Y/N)?  Choosing Y will permanently delete the contents.' % args.dest
+        result = str.upper(sys.stdin.read(1))
+        if result == 'N':
+            print 'Unable to copy files to the destination.  The destination already exists.'
+            sys.exit(1)
+        elif result == 'Y':
+            break
+    shutil.rmtree(args.dest)
+
+os.mkdir(args.dest)
+copy_distro(args.dest, 'x86', args.source, None)
+copy_distro(args.dest, 'x64', args.source, 'amd64')
diff --git a/lldb/source/Interpreter/ScriptInterpreterPython.cpp b/lldb/source/Interpreter/ScriptInterpreterPython.cpp
index 099cbaa0ced..7ecef310b5c 100644
--- a/lldb/source/Interpreter/ScriptInterpreterPython.cpp
+++ b/lldb/source/Interpreter/ScriptInterpreterPython.cpp
@@ -3024,6 +3024,9 @@ ScriptInterpreterPython::InitializePrivate ()
     TerminalState stdin_tty_state;
     stdin_tty_state.Save(STDIN_FILENO, false);
 
+#if defined(LLDB_PYTHON_HOME)
+    Py_SetPythonHome(LLDB_PYTHON_HOME);
+#endif
     PyGILState_STATE gstate;
     Log *log (lldb_private::GetLogIfAllCategoriesSet (LIBLLDB_LOG_SCRIPT | LIBLLDB_LOG_VERBOSE));
     bool threads_already_initialized = false;
diff --git a/lldb/www/build.html b/lldb/www/build.html
index a8a7dd780c7..6d294ae3a20 100755
--- a/lldb/www/build.html
+++ b/lldb/www/build.html
@@ -81,7 +81,8 @@
                 </p>
                 <p>
                   Note that if you plan to do both debug and release builds of LLDB, you will need to
-                  compile both debug and release builds of Python.
+                  compile both debug and release builds of Python.  The same applies if you plan to build
+                  both x86 and x64 configurations of LLDB
                 </p>
               </li>
               <li>
@@ -93,56 +94,20 @@
                   folder.
                 </p>
               </li>
-              <li><p>Install GnuWin32, making sure &lt;GnuWin32 install dir&gt;\bin is added to your PATH environment variable.</p></li>
-              <li><p>Install SWIG for Windows, making sure &lt;SWIG install dir&gt; is added to your PATH environment variable.</p></li>
-              <li>
-                <p>(Optional) Create a file somewhere on your disk called llvm_env.bat and add the following code to it:</p>
-                <code>
-                  @ECHO OFF<br /><br />
-                  IF "%1" == "build" goto do_build<br />
-                  IF "%1" == "dev" goto do_dev<br />
-                  ECHO Unknown option, expected "build" or "dev"<br />
-                  goto do_end<br /><br />
-                  :do_build<br />
-                  ECHO Initializing MSVC Build Environment...<br />
-                  CALL "c:\Program Files (x86)\Microsoft Visual Studio 12.0\vc\vcvarsall.bat"<br />
-                  ECHO Initializing Python environment...<br />
-                  set PYTHONPATH=&lt;python src dir&gt;\Lib;&lt;cmake gen dir&gt;\lib\site-packages<br />
-                  set PATH=%PATH%;&lt;python src dir&gt;\PCbuild<br />
-                  goto do_end<br /><br />
-                  :do_dev<br />
-                  set PYTHONPATH=<br />
-                  goto do_end<br />
-                  :do_end<br />
-                </code>
-              </li>
               <li>
                 <p>
-                  (Optional) To make the process of setting up the environment before building more convenient, you can
-                  optionally create a shortcut with the following properties:
+                  Run lldb/scripts/install_custom_python.py so to "install" your custom build of Python to a
+                  canonical directory structure.
                 </p>
-                <code>%windir%\system32\cmd.exe /K &lt;path-to-llvm_env.bat&gt; build</code>
               </li>
+              <li><p>Install GnuWin32, making sure &lt;GnuWin32 install dir&gt;\bin is added to your PATH environment variable.</p></li>
+              <li><p>Install SWIG for Windows, making sure &lt;SWIG install dir&gt; is added to your PATH environment variable.</p></li>
             </ol>
             <h2>Building LLDB</h2>
             <p>
               Any command prompt from which you build LLDB should have a valid Visual Studio environment setup.
               This means you should run vcvarsall.bat or open an appropriate Visual Studio Command Prompt
-              corresponding to the version you wish to use.  Additionally, in order for LLDB to be able to locate
-              Python to link against, you will need to set up your PYTHONPATH environment variable to contain two
-              additional values:
-            </p>
-            <ol>
-              <li>&lt;python src dir&gt;\Lib</li>
-              <li>&lt;folder where CMake build files are generated&gt;\lib\site-packages</li>
-            </ol>
-            <p>
-              The first allows your custom built python installation to locate its system libraries, and
-              the second allows Python to locate the LLDB extension module.
-            </p>
-            <p>
-              Steps 6 and 7 of <a href="#WindowsPreliminaries">Preliminaries</a> describe a method for simplifying
-              this setup.
+              corresponding to the version you wish to use.
             </p>
             <p>Finally, when you are ready to build LLDB, generate CMake with the following command line:</p>
             <code>cmake -G Ninja &lt;cmake variables&gt; &lt;path to root of llvm src tree&gt;</code>
@@ -159,16 +124,17 @@
                 a crash, rather than having to reproduce a failure or use a crash dump.
               </li>
               <li>
-                <b>PYTHON_LIBRARY</b>: Path to python27.lib.  If doing a debug build, note that the library is called
-                python27_d.lib.  Generally this should be set to <div align="center">&lt;python src dir&gt;\PCBuild\python27(_d).lib</div>
-              </li>
-              <li>
-                <b>PYTHON_INCLUDE_DIR</b>: Path to python include directory.  Generally this should be set to
-                <div align="center">&lt;python src dir&gt;\Include</div>
+                <b>PYTHON_HOME</b> (Required): Path the folder you specified in the --dest argument to install_custom_python.py.
+                Note that install_custom_python.py will create x86 and x64 subdirectories under this folder.  PYTHON_HOME should
+                refer to the correct architecture-specific folder.
               </li>
               <li>
-                <b>PYTHON_EXECUTABLE</b>: Path to python.exe.  If doing a debug build of LLDB, note that the executable
-                is called python_d.exe.  Generally this should be set to <div align="center">&lt;python src dir&gt;\PCBuild\python(_d).exe</div>
+                <b>LLDB_RELOCATABLE_PYTHON</b> (Default=0): When this is 0, LLDB will bind statically to the location specified
+                in the PYTHON_HOME CMake variable, ignoring any value of PYTHONHOME set in the environment.  This is most useful for
+                developers who simply want to run LLDB after they build it.  If you wish to move a build of LLDB to a different
+                machine where Python will be in a different location, setting LLDB_RELOCATABLE_PYTHON to 1 will cause Python to
+                use its default mechanism for finding the python installation at runtime (looking for installed Pythons, or using
+                the PYTHONHOME environment variable if it is specified).
               </li>
             </ul>
           </div>