cpp11-migrate: Write header replacements to disk

Header replacements are now written to disk in YAML format for an external tool
to merge. A unique file will be created in the same directory as the header
with all replacements that came from a source file that included the header
file. The YAML file will have:
 - Name of the file
 - Transform ID that generated the replacement
 - Offset
 - Length
 - Replacement text

Any tool reading these replacements should read them using the TransformDocument
struct.

Differential Revision: http://llvm-reviews.chandlerc.com/D1142

llvm-svn: 187204

6 files changed, 258 insertions, 45 deletions

diff --git a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp
index 58372b60ce7..83678e986b0 100644
--- a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp
+++ b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp
@@ -23,16 +23,26 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
+#include <algorithm>
 
 using namespace clang;
 using namespace clang::tooling;
 
+void HeaderOverride::recordReplacements(
+    llvm::StringRef TransformID, const clang::tooling::Replacements &Replaces) {
+  TransformReplacements TR;
+  TR.TransformID = TransformID;
+  TR.GeneratedReplacements.resize(Replaces.size());
+  std::copy(Replaces.begin(), Replaces.end(), TR.GeneratedReplacements.begin());
+  TransformReplacementsDoc.Replacements.push_back(TR);
+}
+
 SourceOverrides::SourceOverrides(llvm::StringRef MainFileName,
                                  bool TrackChanges)
     : MainFileName(MainFileName), TrackChanges(TrackChanges) {}
 
-void
-SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces) {
+void SourceOverrides::applyReplacements(tooling::Replacements &Replaces,
+                                        llvm::StringRef TransformName) {
   llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> DiagOpts(
       new DiagnosticOptions());
   DiagnosticsEngine Diagnostics(
@@ -40,11 +50,12 @@ SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces) {
       DiagOpts.getPtr());
   FileManager Files((FileSystemOptions()));
   SourceManager SM(Diagnostics, Files);
-  applyReplacements(Replaces, SM);
+  applyReplacements(Replaces, SM, TransformName);
 }
 
-void SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces,
-                                        clang::SourceManager &SM) {
+void SourceOverrides::applyReplacements(tooling::Replacements &Replaces,
+                                        SourceManager &SM,
+                                        llvm::StringRef TransformName) {
   applyOverrides(SM);
 
   Rewriter Rewrites(SM, LangOptions());
@@ -56,12 +67,6 @@ void SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces,
   if (!Success)
     llvm::errs() << "error: failed to apply some replacements.";
 
-  applyRewrites(Rewrites);
-  if (TrackChanges)
-    adjustChangedRanges(Replaces);
-}
-
-void SourceOverrides::applyRewrites(Rewriter &Rewrites) {
   std::string ResultBuf;
 
   for (Rewriter::buffer_iterator I = Rewrites.buffer_begin(),
@@ -89,36 +94,50 @@ void SourceOverrides::applyRewrites(Rewriter &Rewrites) {
     // will be stored as well for later output to disk. Applying replacements
     // in memory will always be necessary as the source goes down the transform
     // pipeline.
-
     HeaderOverride &HeaderOv = Headers[FileName];
-    HeaderOv.FileOverride.swap(ResultBuf);
     // "Create" HeaderOverride if not already existing
-    if (HeaderOv.FileName.empty())
-      HeaderOv.FileName = FileName;
+    if (HeaderOv.getFileName().empty())
+      HeaderOv = HeaderOverride(FileName);
+
+    HeaderOv.swapContentOverride(ResultBuf);
   }
-}
 
-void SourceOverrides::adjustChangedRanges(const Replacements &Replaces) {
-  // Start by grouping replacements by file name
+  // Separate replacements to header files
   Replacements MainFileReplaces;
-  llvm::StringMap<Replacements> HeadersReplaces;
-
-  for (Replacements::iterator I = Replaces.begin(), E = Replaces.end(); I != E;
-       ++I) {
+  ReplacementsMap HeadersReplaces;
+  for (Replacements::const_iterator I = Replaces.begin(), E = Replaces.end();
+      I != E; ++I) {
     llvm::StringRef ReplacementFileName = I->getFilePath();
 
-    if (ReplacementFileName == MainFileName)
+    if (ReplacementFileName == MainFileName) {
       MainFileReplaces.insert(*I);
-    else
-      HeadersReplaces[ReplacementFileName].insert(*I);
+      continue;
+    }
+
+    HeadersReplaces[ReplacementFileName].insert(*I);
   }
 
-  // Then adjust the changed ranges for each individual file
-  MainFileChanges.adjustChangedRanges(Replaces);
-  for (llvm::StringMap<Replacements>::iterator I = HeadersReplaces.begin(),
-                                               E = HeadersReplaces.end();
+  // Record all replacements to headers.
+  for (ReplacementsMap::const_iterator I = HeadersReplaces.begin(),
+                                       E = HeadersReplaces.end();
+       I != E; ++I) {
+    HeaderOverride &HeaderOv = Headers[I->getKey()];
+    HeaderOv.recordReplacements(TransformName, I->getValue());
+  }
+
+  if (TrackChanges)
+    adjustChangedRanges(MainFileReplaces, HeadersReplaces);
+}
+
+void
+SourceOverrides::adjustChangedRanges(const Replacements &MainFileReplaces,
+                                     const ReplacementsMap &HeadersReplaces) {
+  // Adjust the changed ranges for each individual file
+  MainFileChanges.adjustChangedRanges(MainFileReplaces);
+  for (ReplacementsMap::const_iterator I = HeadersReplaces.begin(),
+                                       E = HeadersReplaces.end();
        I != E; ++I) {
-    Headers[I->getKey()].Changes.adjustChangedRanges(I->getValue());
+    Headers[I->getKey()].adjustChangedRanges(I->getValue());
   }
 }
 
@@ -131,11 +150,11 @@ void SourceOverrides::applyOverrides(SourceManager &SM) const {
 
   for (HeaderOverrides::const_iterator I = Headers.begin(), E = Headers.end();
        I != E; ++I) {
-    assert(!I->second.FileOverride.empty() &&
+    assert(!I->second.getContentOverride().empty() &&
            "Header override should not be empty!");
     SM.overrideFileContents(
-        FM.getFile(I->second.FileName),
-        llvm::MemoryBuffer::getMemBuffer(I->second.FileOverride));
+        FM.getFile(I->second.getFileName()),
+        llvm::MemoryBuffer::getMemBuffer(I->second.getContentOverride()));
   }
 }
 
diff --git a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h
index 2bb3daea7b0..3bb1acb3823 100644
--- a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h
+++ b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h
@@ -16,6 +16,7 @@
 #ifndef CPP11_MIGRATE_FILE_OVERRIDES_H
 #define CPP11_MIGRATE_FILE_OVERRIDES_H
 
+#include "Core/ReplacementsYaml.h"
 #include "clang/Tooling/Refactoring.h"
 #include "llvm/ADT/StringMap.h"
 
@@ -58,13 +59,52 @@ private:
 };
 
 /// \brief Container for storing override information for a single headers.
-struct HeaderOverride {
+class HeaderOverride {
+public:
+  /// \brief Constructors.
+  /// @{
   HeaderOverride() {}
-  HeaderOverride(llvm::StringRef FileName) : FileName(FileName) {}
+  HeaderOverride(llvm::StringRef FileName) {
+    TransformReplacementsDoc.FileName = FileName;
+  }
+  /// @}
+
+  /// \brief Getter for FileName.
+  llvm::StringRef getFileName() const {
+    return TransformReplacementsDoc.FileName;
+  }
+
+  /// \brief Accessor for ContentOverride.
+  /// @{
+  llvm::StringRef getContentOverride() const { return ContentOverride; }
+  void setContentOverride(const llvm::StringRef S) { ContentOverride = S; }
+  /// @}
+
+  /// \brief Getter for Changes.
+  const ChangedRanges &getChanges() const { return Changes; }
 
-  std::string FileName;
-  std::string FileOverride;
+  /// \brief Swaps the content of ContentOverride with \param S
+  void swapContentOverride(std::string &S) { ContentOverride.swap(S); }
+
+  /// \brief Getter for TransformReplacementsDoc.
+  const TransformDocument &getTransformReplacementsDoc() const {
+    return TransformReplacementsDoc;
+  }
+
+  /// \brief Stores the replacements made by a transform to the header this
+  /// object represents.
+  void recordReplacements(llvm::StringRef TransformID,
+                          const clang::tooling::Replacements &Replaces);
+
+  /// \brief Helper function to adjust the changed ranges.
+  void adjustChangedRanges(const clang::tooling::Replacements &Replaces) {
+    Changes.adjustChangedRanges(Replaces);
+  }
+
+private:
+  std::string ContentOverride;
   ChangedRanges Changes;
+  TransformDocument TransformReplacementsDoc;
 };
 
 /// \brief Container mapping header file names to override information.
@@ -100,8 +140,10 @@ public:
   /// \param Replaces The replacements to apply.
   /// \param SM A user provided SourceManager to be used when applying rewrites.
   void applyReplacements(clang::tooling::Replacements &Replaces,
-                         clang::SourceManager &SM);
-  void applyReplacements(clang::tooling::Replacements &Replaces);
+                         clang::SourceManager &SM,
+                         llvm::StringRef TransformName);
+  void applyReplacements(clang::tooling::Replacements &Replaces,
+                         llvm::StringRef TransformName);
 
   /// \brief Convenience function for applying this source's overrides to
   /// the given SourceManager.
@@ -118,13 +160,16 @@ public:
   /// @}
 
 private:
+  typedef llvm::StringMap<clang::tooling::Replacements> ReplacementsMap;
+
   /// \brief Flatten the Rewriter buffers of \p Rewrite and store results as
   /// file content overrides.
   void applyRewrites(clang::Rewriter &Rewrite);
 
   /// \brief Adjust the changed ranges to reflect the parts of the files that
   /// have been replaced.
-  void adjustChangedRanges(const clang::tooling::Replacements &Replaces);
+  void adjustChangedRanges(const clang::tooling::Replacements &Replaces,
+                           const ReplacementsMap &HeadersReplaces);
 
   const std::string MainFileName;
   std::string MainFileOverride;
diff --git a/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp b/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp
index 3fd77612bbf..e50206fc0f7 100644
--- a/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp
+++ b/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp
@@ -47,10 +47,10 @@ void Reformatter::reformatChanges(SourceOverrides &Overrides,
        I != E; ++I) {
     const HeaderOverride &Header = I->getValue();
     const tooling::Replacements &HeaderReplaces =
-        reformatSingleFile(Header.FileName, Header.Changes, SM);
+        reformatSingleFile(Header.getFileName(), Header.getChanges(), SM);
     Replaces.insert(HeaderReplaces.begin(), HeaderReplaces.end());
   }
-  Overrides.applyReplacements(Replaces, SM);
+  Overrides.applyReplacements(Replaces, SM, "reformatter");
 }
 
 tooling::Replacements Reformatter::reformatSingleFile(
diff --git a/clang-tools-extra/cpp11-migrate/Core/ReplacementsYaml.h b/clang-tools-extra/cpp11-migrate/Core/ReplacementsYaml.h
new file mode 100644
index 00000000000..0374aae9aaf
--- /dev/null
+++ b/clang-tools-extra/cpp11-migrate/Core/ReplacementsYaml.h
@@ -0,0 +1,115 @@
+//===-- Core/ReplacementsYaml.h ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides structs to store the migrator specific header
+/// replacements and the YAML traits for Replacements.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef CPP11_MIGRATE_REPLACEMENTS_YAML_H
+#define CPP11_MIGRATE_REPLACEMENTS_YAML_H
+
+#include "clang/Tooling/Refactoring.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <vector>
+#include <string>
+
+/// \brief A replacement struct to store the transform ID and the replacement.
+struct TransformReplacements {
+  std::string TransformID;
+  std::vector<clang::tooling::Replacement> GeneratedReplacements;
+};
+
+/// \brief The top-level YAML document that contains the name of the file and
+/// the TransformReplacements.
+struct TransformDocument {
+  std::string FileName;
+  std::vector<TransformReplacements> Replacements;
+};
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(clang::tooling::Replacement)
+LLVM_YAML_IS_SEQUENCE_VECTOR(TransformReplacements)
+
+namespace llvm {
+namespace yaml {
+
+/// \brief ScalarTraits to read/write std::string objects.
+template <>
+struct ScalarTraits<std::string> {
+  static void output(const std::string &Val, void *, llvm::raw_ostream &Out) {
+    // We need to put quotes around the string to make sure special characters
+    // in the string is not treated as YAML tokens.
+    std::string NormalizedVal = std::string("\"") + Val + std::string("\"");
+    Out << NormalizedVal;
+  }
+
+  static StringRef input(StringRef Scalar, void *, std::string &Val) {
+    Val = Scalar;
+    return StringRef();
+  }
+};
+
+/// \brief Specialized MappingTraits for Repleacements to be converted to/from
+/// a YAML File.
+template <>
+struct MappingTraits<clang::tooling::Replacement> {
+  /// \brief Normalize clang::tooling::Replacement to provide direct access to
+  /// its members.
+  struct NormalizedReplacement {
+    NormalizedReplacement(const IO &)
+        : FilePath(""), Offset(0), Length(0), ReplacementText("") {}
+
+    NormalizedReplacement(const IO &, const clang::tooling::Replacement &R)
+        : FilePath(R.getFilePath()), Offset(R.getOffset()),
+          Length(R.getLength()), ReplacementText(R.getReplacementText()) {}
+
+    clang::tooling::Replacement denormalize(const IO &) {
+      return clang::tooling::Replacement(FilePath, Offset, Length,
+                                         ReplacementText);
+    }
+
+    std::string FilePath;
+    unsigned int Offset;
+    unsigned int Length;
+    std::string ReplacementText;
+  };
+
+  static void mapping(IO &Io, clang::tooling::Replacement &R) {
+    MappingNormalization<NormalizedReplacement, clang::tooling::Replacement>
+        Keys(Io, R);
+    Io.mapRequired("Offset", Keys->Offset);
+    Io.mapRequired("Length", Keys->Length);
+    Io.mapRequired("ReplacementText", Keys->ReplacementText);
+  }
+};
+
+/// \brief Specialized MappingTraits for TransformRepleacements to be converted
+/// to/from a YAML File.
+template <>
+struct MappingTraits<TransformReplacements> {
+  static void mapping(IO &Io, TransformReplacements &R) {
+    Io.mapRequired("TransformID", R.TransformID);
+    Io.mapRequired("Replacements", R.GeneratedReplacements);
+  }
+};
+
+/// \brief Specialized MappingTraits for TransformDocument to be converted
+/// to/from a YAML File.
+template <>
+struct MappingTraits<TransformDocument> {
+  static void mapping(IO &Io, TransformDocument &TD) {
+    Io.mapRequired("FileName", TD.FileName);
+    Io.mapRequired("Transforms", TD.Replacements);
+  }
+};
+} // end namespace yaml
+} // end namespace llvm
+
+#endif // CPP11_MIGRATE_REPLACEMENTS_YAML_H
diff --git a/clang-tools-extra/cpp11-migrate/Core/Transform.cpp b/clang-tools-extra/cpp11-migrate/Core/Transform.cpp
index ac3375316bb..7338235fb33 100644
--- a/clang-tools-extra/cpp11-migrate/Core/Transform.cpp
+++ b/clang-tools-extra/cpp11-migrate/Core/Transform.cpp
@@ -117,7 +117,7 @@ bool Transform::handleBeginSource(CompilerInstance &CI, StringRef Filename) {
 void Transform::handleEndSource() {
   if (!getReplacements().empty()) {
     SourceOverrides &SO = Overrides->getOrCreate(CurrentSource);
-    SO.applyReplacements(getReplacements());
+    SO.applyReplacements(getReplacements(), getName());
   }
 
   if (Options().EnableTiming)
diff --git a/clang-tools-extra/cpp11-migrate/tool/Cpp11Migrate.cpp b/clang-tools-extra/cpp11-migrate/tool/Cpp11Migrate.cpp
index 2ee4a069ff3..9f8049aeea3 100644
--- a/clang-tools-extra/cpp11-migrate/tool/Cpp11Migrate.cpp
+++ b/clang-tools-extra/cpp11-migrate/tool/Cpp11Migrate.cpp
@@ -21,11 +21,18 @@
 #include "Core/Transform.h"
 #include "Core/Transforms.h"
 #include "Core/Reformatting.h"
+#include "Core/ReplacementsYaml.h"
+#include "LoopConvert/LoopConvert.h"
+#include "UseNullptr/UseNullptr.h"
+#include "UseAuto/UseAuto.h"
+#include "AddOverride/AddOverride.h"
+#include "ReplaceAutoPtr/ReplaceAutoPtr.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Signals.h"
+#include "llvm/Support/FileSystem.h"
 
 namespace cl = llvm::cl;
 using namespace clang;
@@ -251,13 +258,40 @@ int main(int argc, const char **argv) {
     for (HeaderOverrides::const_iterator HeaderI = Overrides.headers_begin(),
                                          HeaderE = Overrides.headers_end();
          HeaderI != HeaderE; ++HeaderI) {
-      assert(!HeaderI->second.FileOverride.empty() &&
+      assert(!HeaderI->second.getContentOverride().empty() &&
              "A header override should not be empty");
       std::string ErrorInfo;
       std::string HeaderFileName = HeaderI->getKey();
       llvm::raw_fd_ostream HeaderStream(HeaderFileName.c_str(), ErrorInfo,
                                         llvm::sys::fs::F_Binary);
-      HeaderStream << HeaderI->second.FileOverride;
+      if (!ErrorInfo.empty()) {
+        llvm::errs() << "Error opening file: " << ErrorInfo << "\n";
+        continue;
+      }
+      HeaderStream << HeaderI->second.getContentOverride();
+
+      // Replacements for header files need to be written in a YAML file for
+      // every transform and will be merged together with an external tool.
+      llvm::SmallString<128> ReplacementsHeaderName;
+      llvm::SmallString<64> Error;
+      bool Result = generateReplacementsFileName(I->getKey(), HeaderFileName,
+                                                 ReplacementsHeaderName, Error);
+      if (!Result) {
+        llvm::errs() << "Failed to generate replacements filename:" << Error
+                     << "\n";
+        continue;
+      }
+
+      llvm::raw_fd_ostream ReplacementsFile(ReplacementsHeaderName.c_str(),
+                                            ErrorInfo,
+                                            llvm::sys::fs::F_Binary);
+      if (!ErrorInfo.empty()) {
+        llvm::errs() << "Error opening file: " << ErrorInfo << "\n";
+        continue;
+      }
+      llvm::yaml::Output YAML(ReplacementsFile);
+      YAML << const_cast<TransformDocument &>(
+                  HeaderI->getValue().getTransformReplacementsDoc());
     }
   }