Added support for extracting headers from module maps as a source for the header list.

llvm-svn: 229692

6 files changed, 325 insertions, 17 deletions

diff --git a/clang-tools-extra/docs/ModularizeUsage.rst b/clang-tools-extra/docs/ModularizeUsage.rst
index 90b2d6a8d0f..c93a72b3a22 100644
--- a/clang-tools-extra/docs/ModularizeUsage.rst
+++ b/clang-tools-extra/docs/ModularizeUsage.rst
@@ -2,13 +2,22 @@
 Modularize Usage
 ================
 
-``modularize [<modularize-options>] <include-files-list>[,<include-files-list>]*
+``modularize [<modularize-options>] [<module-map>|<include-files-list>]*
 [<front-end-options>...]``
 
 ``<modularize-options>`` is a place-holder for options
 specific to modularize, which are described below in
 `Modularize Command Line Options`.
 
+``<module-map>`` specifies the path of a file name for an
+existing module map.  The module map must be well-formed in
+terms of syntax.  Modularize will extract the header file names
+from the map.  Only normal headers are checked, assuming headers
+marked "private", "textual", or "exclude" are not to be checked
+as a top-level include, assuming they either are included by
+other headers which are checked, or they are not suitable for
+modules.
+
 ``<include-files-list>`` specifies the path of a file name for a
 file containing the newline-separated list of headers to check
 with respect to each other. Lines beginning with '#' and empty
diff --git a/clang-tools-extra/modularize/Modularize.cpp b/clang-tools-extra/modularize/Modularize.cpp
index c673ceadd9b..4578185efa5 100644
--- a/clang-tools-extra/modularize/Modularize.cpp
+++ b/clang-tools-extra/modularize/Modularize.cpp
@@ -8,25 +8,40 @@
 //===----------------------------------------------------------------------===//
 //
 // This file implements a tool that checks whether a set of headers provides
-// the consistent definitions required to use modules. For example, it detects
-// whether the same entity (say, a NULL macro or size_t typedef) is defined in
-// multiple headers or whether a header produces different definitions under
+// the consistent definitions required to use modules.  It can also check an
+// existing module map for full coverage of the headers in a directory tree.
+//
+// For example, in examining headers, it detects whether the same entity
+// (say, a NULL macro or size_t typedef) is defined in multiple headers
+// or whether a header produces different definitions under
 // different circumstances. These conditions cause modules built from the
 // headers to behave poorly, and should be fixed before introducing a module
 // map.
 //
-// Modularize takes as argument one or more file names for files containing
-// newline-separated lists of headers to check with respect to each other.
+// Modularize takes as input either one or more module maps (by default,
+// "module.modulemap") or one or more text files contatining lists of headers
+// to check.
+//
+// In the case of a module map, the module map must be well-formed in
+// terms of syntax.  Modularize will extract the header file names
+// from the map.  Only normal headers are checked, assuming headers
+// marked "private", "textual", or "exclude" are not to be checked
+// as a top-level include, assuming they either are included by
+// other headers which are checked, or they are not suitable for
+// modules.
+//
+// In the case of a file list, the list is a newline-separated list of headers
+// to check with respect to each other.
 // Lines beginning with '#' and empty lines are ignored.
 // Header file names followed by a colon and other space-separated
 // file names will include those extra files as dependencies.
 // The file names can be relative or full paths, but must be on the
 // same line.
 //
-// Modularize also accepts regular front-end arguments.
+// Modularize also accepts regular clang front-end arguments.
 //
-// Usage:   modularize [-prefix (optional header path prefix)]
-//   (include-files_list)[,(include-files_list)]* [(front-end-options) ...]
+// Usage:   modularize [(modularize options)]
+//   [(include-files_list)|(module map)]+ [(front-end-options) ...]
 //
 // Options:
 //    -prefix (optional header path prefix)
diff --git a/clang-tools-extra/modularize/ModularizeUtilities.cpp b/clang-tools-extra/modularize/ModularizeUtilities.cpp
index d3c48c9f136..03328c80193 100644
--- a/clang-tools-extra/modularize/ModularizeUtilities.cpp
+++ b/clang-tools-extra/modularize/ModularizeUtilities.cpp
@@ -14,22 +14,48 @@
 //===----------------------------------------------------------------------===//
 
 #include "ModularizeUtilities.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Driver/Options.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
 #include "llvm/ADT/SmallString.h"

 #include "llvm/Support/FileUtilities.h"

 #include "llvm/Support/MemoryBuffer.h"

 #include "llvm/Support/Path.h"

 #include "llvm/Support/raw_ostream.h"

 
+using namespace clang;
 using namespace llvm;
 using namespace Modularize;
 
+// Subclass TargetOptions so we can construct it inline with
+// the minimal option, the triple.
+class ModuleMapTargetOptions : public clang::TargetOptions {
+public:
+  ModuleMapTargetOptions() { Triple = llvm::sys::getDefaultTargetTriple(); }
+};
+
 // ModularizeUtilities class implementation.
 
 // Constructor.
 ModularizeUtilities::ModularizeUtilities(std::vector<std::string> &InputPaths,
                                          llvm::StringRef Prefix)
-    : InputFilePaths(InputPaths),
-      HeaderPrefix(Prefix) {}
+  : InputFilePaths(InputPaths),
+    HeaderPrefix(Prefix),
+    // Init clang stuff needed for loading the module map and preprocessing.
+    LangOpts(new LangOptions()), DiagIDs(new DiagnosticIDs()),
+    DiagnosticOpts(new DiagnosticOptions()),
+    DC(llvm::errs(), DiagnosticOpts.get()),
+    Diagnostics(
+    new DiagnosticsEngine(DiagIDs, DiagnosticOpts.get(), &DC, false)),
+    TargetOpts(new ModuleMapTargetOptions()),
+    Target(TargetInfo::CreateTargetInfo(*Diagnostics, TargetOpts)),
+    FileMgr(new FileManager(FileSystemOpts)),
+    SourceMgr(new SourceManager(*Diagnostics, *FileMgr, false)),
+    HeaderSearchOpts(new HeaderSearchOptions()),
+    HeaderInfo(new HeaderSearch(HeaderSearchOpts, *SourceMgr, *Diagnostics,
+    *LangOpts, Target.get())) {
+}
 
 // Create instance of ModularizeUtilities, to simplify setting up
 // subordinate objects.
@@ -42,11 +68,22 @@ ModularizeUtilities *ModularizeUtilities::createModularizeUtilities(
 // Load all header lists and dependencies.
 std::error_code ModularizeUtilities::loadAllHeaderListsAndDependencies() {
   typedef std::vector<std::string>::iterator Iter;
+  // For each input file.
   for (Iter I = InputFilePaths.begin(), E = InputFilePaths.end(); I != E; ++I) {
-    if (std::error_code EC = loadSingleHeaderListsAndDependencies(*I)) {
-      errs() << "modularize: error: Unable to get header list '" << *I
-        << "': " << EC.message() << '\n';
-      return EC;
+    llvm::StringRef InputPath = *I;
+    // If it's a module map.
+    if (InputPath.endswith(".modulemap")) {
+      // Load the module map.
+      if (std::error_code EC = loadModuleMap(InputPath))
+        return EC;
+    }
+    else {
+      // Else we assume it's a header list and load it.
+      if (std::error_code EC = loadSingleHeaderListsAndDependencies(InputPath)) {
+        errs() << "modularize: error: Unable to get header list '" << InputPath
+          << "': " << EC.message() << '\n';
+        return EC;
+      }
     }
   }
   return std::error_code();
@@ -125,6 +162,156 @@ std::error_code ModularizeUtilities::loadSingleHeaderListsAndDependencies(
   return std::error_code();
 }
 
+// Load single module map and extract header file list.
+std::error_code ModularizeUtilities::loadModuleMap(
+    llvm::StringRef InputPath) {
+  // Get file entry for module.modulemap file.
+  const FileEntry *ModuleMapEntry =
+    SourceMgr->getFileManager().getFile(InputPath);
+
+  // return error if not found.
+  if (!ModuleMapEntry) {
+    llvm::errs() << "error: File \"" << InputPath << "\" not found.\n";
+    return std::error_code(1, std::generic_category());
+  }
+
+  // Because the module map parser uses a ForwardingDiagnosticConsumer,
+  // which doesn't forward the BeginSourceFile call, we do it explicitly here.
+  DC.BeginSourceFile(*LangOpts, nullptr);
+
+  // Figure out the home directory for the module map file.
+  const DirectoryEntry *Dir = ModuleMapEntry->getDir();
+  StringRef DirName(Dir->getName());
+  if (llvm::sys::path::filename(DirName) == "Modules") {
+    DirName = llvm::sys::path::parent_path(DirName);
+    if (DirName.endswith(".framework"))
+      Dir = FileMgr->getDirectory(DirName);
+    // FIXME: This assert can fail if there's a race between the above check
+    // and the removal of the directory.
+    assert(Dir && "parent must exist");
+  }
+
+  std::unique_ptr<ModuleMap> ModMap;
+  ModMap.reset(new ModuleMap(*SourceMgr, *Diagnostics, *LangOpts,
+    Target.get(), *HeaderInfo));
+
+  // Parse module.modulemap file into module map.
+  if (ModMap->parseModuleMapFile(ModuleMapEntry, false, Dir)) {
+    return std::error_code(1, std::generic_category());
+  }
+
+  // Do matching end call.
+  DC.EndSourceFile();
+
+  if (!collectModuleMapHeaders(ModMap.get()))
+    return std::error_code(1, std::generic_category());
+
+  // Save module map.
+  ModuleMaps.push_back(std::move(ModMap));
+
+  return std::error_code();
+}
+
+// Collect module map headers.
+// Walks the modules and collects referenced headers into
+// ModuleMapHeadersSet.
+bool ModularizeUtilities::collectModuleMapHeaders(clang::ModuleMap *ModMap) {
+  for (ModuleMap::module_iterator I = ModMap->module_begin(),
+    E = ModMap->module_end();
+    I != E; ++I) {
+    if (!collectModuleHeaders(*I->second))
+      return false;
+  }
+  return true;
+}
+
+// Collect referenced headers from one module.
+// Collects the headers referenced in the given module into
+// HeaderFileNames and ModuleMapHeadersSet.
+bool ModularizeUtilities::collectModuleHeaders(const Module &Mod) {
+
+  // Ignore explicit modules because they often have dependencies
+  // we can't know.
+  if (Mod.IsExplicit)
+    return true;
+
+  // Treat headers in umbrella directory as dependencies.
+  DependentsVector UmbrellaDependents;
+
+  // Recursively do submodules.
+  for (Module::submodule_const_iterator MI = Mod.submodule_begin(),
+      MIEnd = Mod.submodule_end();
+      MI != MIEnd; ++MI)
+    collectModuleHeaders(**MI);
+
+  if (const FileEntry *UmbrellaHeader = Mod.getUmbrellaHeader()) {
+    std::string HeaderPath = getCanonicalPath(UmbrellaHeader->getName());
+    // Collect umbrella header.
+    HeaderFileNames.push_back(HeaderPath);
+    ModuleMapHeadersSet.insert(HeaderPath);
+
+    // FUTURE: When needed, umbrella header header collection goes here.
+  }
+  else if (const DirectoryEntry *UmbrellaDir = Mod.getUmbrellaDir()) {
+    // If there normal headers, assume these are umbrellas and skip collection.
+    if (Mod.Headers->size() == 0) {
+      // Collect headers in umbrella directory.
+      if (!collectUmbrellaHeaders(UmbrellaDir->getName(), UmbrellaDependents))
+        return false;
+    }
+  }
+
+  // We ignore HK_Private, HK_Textual, HK_PrivateTextual, and HK_Excluded,
+  // assuming they are marked as such either because of unsuitability for
+  // modules or because they are meant to be included by another header,
+  // and thus should be ignored by modularize.
+
+  int NormalHeaderCount = Mod.Headers[clang::Module::HK_Normal].size();
+
+  for (int Index = 0; Index < NormalHeaderCount; ++Index) {
+    DependentsVector NormalDependents;
+    // Collect normal header.
+    const clang::Module::Header &Header(
+      Mod.Headers[clang::Module::HK_Normal][Index]);
+    std::string HeaderPath = getCanonicalPath(Header.Entry->getName());
+    HeaderFileNames.push_back(HeaderPath);
+  }
+
+  return true;
+}
+
+// Collect headers from an umbrella directory.
+bool ModularizeUtilities::collectUmbrellaHeaders(StringRef UmbrellaDirName,
+  DependentsVector &Dependents) {
+  // Initialize directory name.
+  SmallString<256> Directory(UmbrellaDirName);
+  // Walk the directory.
+  std::error_code EC;
+  llvm::sys::fs::file_status Status;
+  for (llvm::sys::fs::directory_iterator I(Directory.str(), EC), E; I != E;
+    I.increment(EC)) {
+    if (EC)
+      return false;
+    std::string File(I->path());
+    I->status(Status);
+    llvm::sys::fs::file_type Type = Status.type();
+    // If the file is a directory, ignore the name and recurse.
+    if (Type == llvm::sys::fs::file_type::directory_file) {
+      if (!collectUmbrellaHeaders(File, Dependents))
+        return false;
+      continue;
+    }
+    // If the file does not have a common header extension, ignore it.
+    if (!isHeader(File))
+      continue;
+    // Save header name.
+    std::string HeaderPath = getCanonicalPath(File);
+    ModuleMapHeadersSet.insert(HeaderPath);
+    Dependents.push_back(HeaderPath);
+  }
+  return true;
+}
+
 // Convert header path to canonical form.
 // The canonical form is basically just use forward slashes, and remove "./".
 // \param FilePath The file path, relative to the module map directory.
@@ -137,3 +324,19 @@ std::string ModularizeUtilities::getCanonicalPath(StringRef FilePath) {
     Tmp = Tmp2.substr(2);
   return Tmp;
 }
+
+// Check for header file extension.
+// If the file extension is .h, .inc, or missing, it's
+// assumed to be a header.
+// \param FileName The file name.  Must not be a directory.
+// \returns true if it has a header extension or no extension.
+bool ModularizeUtilities::isHeader(StringRef FileName) {
+  StringRef Extension = llvm::sys::path::extension(FileName);
+  if (Extension.size() == 0)
+    return false;
+  if (Extension.equals_lower(".h"))
+    return true;
+  if (Extension.equals_lower(".inc"))
+    return true;
+  return false;
+}
diff --git a/clang-tools-extra/modularize/ModularizeUtilities.h b/clang-tools-extra/modularize/ModularizeUtilities.h
index aac79d2fd9d..e40493ce039 100644
--- a/clang-tools-extra/modularize/ModularizeUtilities.h
+++ b/clang-tools-extra/modularize/ModularizeUtilities.h
@@ -16,6 +16,18 @@
 #define MODULARIZEUTILITIES_H
 
 #include "Modularize.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/ModuleMap.h"
+#include "clang/Lex/Preprocessor.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSet.h"
 #include <string>
 #include <vector>
 
@@ -34,10 +46,12 @@ public:
 
   // Output data.
 
-  // List of top-level header files.
+  /// List of top-level header files.
   llvm::SmallVector<std::string, 32> HeaderFileNames;
-  // Map of top-level header file dependencies.
+  /// Map of top-level header file dependencies.
   DependencyMap Dependencies;
+  /// Set of all the headers found in the module map.
+  llvm::StringSet<llvm::MallocAllocator> ModuleMapHeadersSet;
 
   // Functions.
 
@@ -61,13 +75,42 @@ public:
   /// \returns std::error_code.
   std::error_code loadAllHeaderListsAndDependencies();
 
+  // Internal.
+
 protected:
+
   /// Load single header list and dependencies.
   /// \param InputPath The input file path.
   /// \returns std::error_code.
   std::error_code loadSingleHeaderListsAndDependencies(
       llvm::StringRef InputPath);
 
+  /// Load single module map and extract header file list.
+  /// \param InputPath The input file path.
+  /// \returns std::error_code.
+  std::error_code loadModuleMap(
+    llvm::StringRef InputPath);
+
+  /// Collect module Map headers.
+  /// Walks the modules and collects referenced headers into
+  /// ModuleMapHeadersSet.
+  /// \param ModMap A loaded module map object.
+  /// \return True if no errors.
+  bool collectModuleMapHeaders(clang::ModuleMap *ModMap);
+
+  /// Collect referenced headers from one module.
+  /// Collects the headers referenced in the given module into
+  /// HeaderFileNames and ModuleMapHeadersSet.
+  /// \param Mod The module reference.
+  /// \return True if no errors.
+  bool collectModuleHeaders(const clang::Module &Mod);
+
+  /// Collect headers from an umbrella directory.
+  /// \param UmbrellaDirName The umbrella directory name.
+  /// \return True if no errors.
+  bool collectUmbrellaHeaders(llvm::StringRef UmbrellaDirName,
+    DependentsVector &Dependents);
+
 public:
 
   // Utility functions.
@@ -78,6 +121,42 @@ public:
   /// \param FilePath The file path.
   /// \returns The file path in canonical form.
   static std::string getCanonicalPath(llvm::StringRef FilePath);
+
+  /// Check for header file extension.
+  /// If the file extension is .h, .inc, or missing, it's
+  /// assumed to be a header.
+  /// \param FileName The file name.  Must not be a directory.
+  /// \returns true if it has a header extension or no extension.
+  static bool isHeader(llvm::StringRef FileName);
+
+  // Internal data.
+
+  /// Options controlling the language variant.
+  std::shared_ptr<clang::LangOptions> LangOpts;
+  /// Diagnostic IDs.
+  const llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs> DiagIDs;
+  /// Options controlling the diagnostic engine.
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> DiagnosticOpts;
+  /// Diagnostic consumer.
+  clang::TextDiagnosticPrinter DC;
+  /// Diagnostic engine.
+  llvm::IntrusiveRefCntPtr<clang::DiagnosticsEngine> Diagnostics;
+  /// Options controlling the target.
+  std::shared_ptr<clang::TargetOptions> TargetOpts;
+  /// Target information.
+  llvm::IntrusiveRefCntPtr<clang::TargetInfo> Target;
+  /// Options controlling the file system manager.
+  clang::FileSystemOptions FileSystemOpts;
+  /// File system manager.
+  llvm::IntrusiveRefCntPtr<clang::FileManager> FileMgr;
+  /// Source manager.
+  llvm::IntrusiveRefCntPtr<clang::SourceManager> SourceMgr;
+  /// Options controlling the \#include directive.
+  llvm::IntrusiveRefCntPtr<clang::HeaderSearchOptions> HeaderSearchOpts;
+  /// Header search manager.
+  std::unique_ptr<clang::HeaderSearch> HeaderInfo;
+  // The loaded module map objects.
+  std::vector<std::unique_ptr<clang::ModuleMap>> ModuleMaps;
 };
 
 } // end namespace Modularize
diff --git a/clang-tools-extra/test/modularize/NoProblems.modularize b/clang-tools-extra/test/modularize/NoProblems.modularize
index c162b63326c..48d3482910f 100644
--- a/clang-tools-extra/test/modularize/NoProblems.modularize
+++ b/clang-tools-extra/test/modularize/NoProblems.modularize
@@ -1,5 +1,6 @@
 # RUN: modularize %s -x c++
 # RUN: modularize -prefix=%p %s -x c++
+# RUN: modularize %S/Inputs/NoProblems.modulemap -x c++
 
 Inputs/SomeTypes.h
 Inputs/SomeDecls.h
diff --git a/clang-tools-extra/test/modularize/ProblemsDuplicate.modularize b/clang-tools-extra/test/modularize/ProblemsDuplicate.modularize
index 35ff6b2109a..301c9cc8c11 100644
--- a/clang-tools-extra/test/modularize/ProblemsDuplicate.modularize
+++ b/clang-tools-extra/test/modularize/ProblemsDuplicate.modularize
@@ -1,4 +1,5 @@
 # RUN: not modularize %s -x c++ 2>&1 | FileCheck %s
+# RUN: not modularize %S/Inputs/ProblemsDuplicate.modulemap -x c++ 2>&1 | FileCheck %s
 
 Inputs/DuplicateHeader1.h
 Inputs/DuplicateHeader2.h