cpp11-migrate: Write header replacements to disk

Another attempt to commit r187204 after windows related problems has
been fixed. Note that changes to this patch reflect the current behavior
of cpp11-migrate.

Header replacements are now written to disk in YAML format for an external tool
to merge. A unique file will be created in the same directory as the header
with all replacements that came from a source file that included the header
file. The YAML file will have:
- Name of the header file
- Name of the source file that included the header file
- Transform ID that generated the replacement
- Offset
- Length
- Replacement text

Any tool reading these replacements should read them using the
HeaderChangeDocument struct.

Differential Revision: http://llvm-reviews.chandlerc.com/D1369

llvm-svn: 188274

5 files changed, 210 insertions, 43 deletions

diff --git a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp
index 58372b60ce7..0f1b0d45f3e 100644
--- a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp
+++ b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.cpp
@@ -23,16 +23,25 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
+#include <algorithm>
 
 using namespace clang;
 using namespace clang::tooling;
 
+void HeaderOverride::recordReplacements(
+    llvm::StringRef TransformID, const clang::tooling::Replacements &Replaces) {
+  HeaderChangeDoc.TransformID = TransformID;
+  HeaderChangeDoc.Replacements.resize(Replaces.size());
+  std::copy(Replaces.begin(), Replaces.end(),
+            HeaderChangeDoc.Replacements.begin());
+}
+
 SourceOverrides::SourceOverrides(llvm::StringRef MainFileName,
                                  bool TrackChanges)
     : MainFileName(MainFileName), TrackChanges(TrackChanges) {}
 
-void
-SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces) {
+void SourceOverrides::applyReplacements(tooling::Replacements &Replaces,
+                                        llvm::StringRef TransformName) {
   llvm::IntrusiveRefCntPtr<clang::DiagnosticOptions> DiagOpts(
       new DiagnosticOptions());
   DiagnosticsEngine Diagnostics(
@@ -40,11 +49,12 @@ SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces) {
       DiagOpts.getPtr());
   FileManager Files((FileSystemOptions()));
   SourceManager SM(Diagnostics, Files);
-  applyReplacements(Replaces, SM);
+  applyReplacements(Replaces, SM, TransformName);
 }
 
-void SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces,
-                                        clang::SourceManager &SM) {
+void SourceOverrides::applyReplacements(tooling::Replacements &Replaces,
+                                        SourceManager &SM,
+                                        llvm::StringRef TransformName) {
   applyOverrides(SM);
 
   Rewriter Rewrites(SM, LangOptions());
@@ -56,12 +66,6 @@ void SourceOverrides::applyReplacements(clang::tooling::Replacements &Replaces,
   if (!Success)
     llvm::errs() << "error: failed to apply some replacements.";
 
-  applyRewrites(Rewrites);
-  if (TrackChanges)
-    adjustChangedRanges(Replaces);
-}
-
-void SourceOverrides::applyRewrites(Rewriter &Rewrites) {
   std::string ResultBuf;
 
   for (Rewriter::buffer_iterator I = Rewrites.buffer_begin(),
@@ -89,36 +93,50 @@ void SourceOverrides::applyRewrites(Rewriter &Rewrites) {
     // will be stored as well for later output to disk. Applying replacements
     // in memory will always be necessary as the source goes down the transform
     // pipeline.
-
     HeaderOverride &HeaderOv = Headers[FileName];
-    HeaderOv.FileOverride.swap(ResultBuf);
     // "Create" HeaderOverride if not already existing
-    if (HeaderOv.FileName.empty())
-      HeaderOv.FileName = FileName;
+    if (HeaderOv.getFileName().empty())
+      HeaderOv = HeaderOverride(FileName, MainFileName);
+
+    HeaderOv.swapContentOverride(ResultBuf);
   }
-}
 
-void SourceOverrides::adjustChangedRanges(const Replacements &Replaces) {
-  // Start by grouping replacements by file name
+  // Separate replacements to header files
   Replacements MainFileReplaces;
-  llvm::StringMap<Replacements> HeadersReplaces;
-
-  for (Replacements::iterator I = Replaces.begin(), E = Replaces.end(); I != E;
-       ++I) {
+  ReplacementsMap HeadersReplaces;
+  for (Replacements::const_iterator I = Replaces.begin(), E = Replaces.end();
+      I != E; ++I) {
     llvm::StringRef ReplacementFileName = I->getFilePath();
 
-    if (ReplacementFileName == MainFileName)
+    if (ReplacementFileName == MainFileName) {
       MainFileReplaces.insert(*I);
-    else
-      HeadersReplaces[ReplacementFileName].insert(*I);
+      continue;
+    }
+
+    HeadersReplaces[ReplacementFileName].insert(*I);
   }
 
-  // Then adjust the changed ranges for each individual file
-  MainFileChanges.adjustChangedRanges(Replaces);
-  for (llvm::StringMap<Replacements>::iterator I = HeadersReplaces.begin(),
-                                               E = HeadersReplaces.end();
+  // Record all replacements to headers.
+  for (ReplacementsMap::const_iterator I = HeadersReplaces.begin(),
+                                       E = HeadersReplaces.end();
+       I != E; ++I) {
+    HeaderOverride &HeaderOv = Headers[I->getKey()];
+    HeaderOv.recordReplacements(TransformName, I->getValue());
+  }
+
+  if (TrackChanges)
+    adjustChangedRanges(MainFileReplaces, HeadersReplaces);
+}
+
+void
+SourceOverrides::adjustChangedRanges(const Replacements &MainFileReplaces,
+                                     const ReplacementsMap &HeadersReplaces) {
+  // Adjust the changed ranges for each individual file
+  MainFileChanges.adjustChangedRanges(MainFileReplaces);
+  for (ReplacementsMap::const_iterator I = HeadersReplaces.begin(),
+                                       E = HeadersReplaces.end();
        I != E; ++I) {
-    Headers[I->getKey()].Changes.adjustChangedRanges(I->getValue());
+    Headers[I->getKey()].adjustChangedRanges(I->getValue());
   }
 }
 
@@ -131,11 +149,11 @@ void SourceOverrides::applyOverrides(SourceManager &SM) const {
 
   for (HeaderOverrides::const_iterator I = Headers.begin(), E = Headers.end();
        I != E; ++I) {
-    assert(!I->second.FileOverride.empty() &&
+    assert(!I->second.getContentOverride().empty() &&
            "Header override should not be empty!");
     SM.overrideFileContents(
-        FM.getFile(I->second.FileName),
-        llvm::MemoryBuffer::getMemBuffer(I->second.FileOverride));
+        FM.getFile(I->second.getFileName()),
+        llvm::MemoryBuffer::getMemBuffer(I->second.getContentOverride()));
   }
 }
 
diff --git a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h
index 2bb3daea7b0..9ca4006077e 100644
--- a/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h
+++ b/clang-tools-extra/cpp11-migrate/Core/FileOverrides.h
@@ -16,6 +16,7 @@
 #ifndef CPP11_MIGRATE_FILE_OVERRIDES_H
 #define CPP11_MIGRATE_FILE_OVERRIDES_H
 
+#include "Core/ReplacementsYaml.h"
 #include "clang/Tooling/Refactoring.h"
 #include "llvm/ADT/StringMap.h"
 
@@ -58,13 +59,54 @@ private:
 };
 
 /// \brief Container for storing override information for a single headers.
-struct HeaderOverride {
+class HeaderOverride {
+public:
+  /// \brief Constructors.
+  /// @{
   HeaderOverride() {}
-  HeaderOverride(llvm::StringRef FileName) : FileName(FileName) {}
+  HeaderOverride(llvm::StringRef HeaderFileName,
+                 llvm::StringRef SourceFileName) {
+    HeaderChangeDoc.HeaderFileName = HeaderFileName;
+    HeaderChangeDoc.SourceFileName = SourceFileName;
+  }
+  /// @}
+
+  /// \brief Getter for FileName.
+  llvm::StringRef getFileName() const {
+    return HeaderChangeDoc.HeaderFileName;
+  }
+
+  /// \brief Accessor for ContentOverride.
+  /// @{
+  llvm::StringRef getContentOverride() const { return ContentOverride; }
+  void setContentOverride(const llvm::StringRef S) { ContentOverride = S; }
+  /// @}
+
+  /// \brief Getter for Changes.
+  const ChangedRanges &getChanges() const { return Changes; }
 
-  std::string FileName;
-  std::string FileOverride;
+  /// \brief Swaps the content of ContentOverride with \param S
+  void swapContentOverride(std::string &S) { ContentOverride.swap(S); }
+
+  /// \brief Getter for HeaderChangeDoc.
+  const HeaderChangeDocument &getHeaderChangeDoc() const {
+    return HeaderChangeDoc;
+  }
+
+  /// \brief Stores the replacements made by a transform to the header this
+  /// object represents.
+  void recordReplacements(llvm::StringRef TransformID,
+                          const clang::tooling::Replacements &Replaces);
+
+  /// \brief Helper function to adjust the changed ranges.
+  void adjustChangedRanges(const clang::tooling::Replacements &Replaces) {
+    Changes.adjustChangedRanges(Replaces);
+  }
+
+private:
+  std::string ContentOverride;
   ChangedRanges Changes;
+  HeaderChangeDocument HeaderChangeDoc;
 };
 
 /// \brief Container mapping header file names to override information.
@@ -100,8 +142,10 @@ public:
   /// \param Replaces The replacements to apply.
   /// \param SM A user provided SourceManager to be used when applying rewrites.
   void applyReplacements(clang::tooling::Replacements &Replaces,
-                         clang::SourceManager &SM);
-  void applyReplacements(clang::tooling::Replacements &Replaces);
+                         clang::SourceManager &SM,
+                         llvm::StringRef TransformName);
+  void applyReplacements(clang::tooling::Replacements &Replaces,
+                         llvm::StringRef TransformName);
 
   /// \brief Convenience function for applying this source's overrides to
   /// the given SourceManager.
@@ -118,13 +162,16 @@ public:
   /// @}
 
 private:
+  typedef llvm::StringMap<clang::tooling::Replacements> ReplacementsMap;
+
   /// \brief Flatten the Rewriter buffers of \p Rewrite and store results as
   /// file content overrides.
   void applyRewrites(clang::Rewriter &Rewrite);
 
   /// \brief Adjust the changed ranges to reflect the parts of the files that
   /// have been replaced.
-  void adjustChangedRanges(const clang::tooling::Replacements &Replaces);
+  void adjustChangedRanges(const clang::tooling::Replacements &Replaces,
+                           const ReplacementsMap &HeadersReplaces);
 
   const std::string MainFileName;
   std::string MainFileOverride;
diff --git a/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp b/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp
index 3fd77612bbf..e50206fc0f7 100644
--- a/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp
+++ b/clang-tools-extra/cpp11-migrate/Core/Reformatting.cpp
@@ -47,10 +47,10 @@ void Reformatter::reformatChanges(SourceOverrides &Overrides,
        I != E; ++I) {
     const HeaderOverride &Header = I->getValue();
     const tooling::Replacements &HeaderReplaces =
-        reformatSingleFile(Header.FileName, Header.Changes, SM);
+        reformatSingleFile(Header.getFileName(), Header.getChanges(), SM);
     Replaces.insert(HeaderReplaces.begin(), HeaderReplaces.end());
   }
-  Overrides.applyReplacements(Replaces, SM);
+  Overrides.applyReplacements(Replaces, SM, "reformatter");
 }
 
 tooling::Replacements Reformatter::reformatSingleFile(
diff --git a/clang-tools-extra/cpp11-migrate/Core/ReplacementsYaml.h b/clang-tools-extra/cpp11-migrate/Core/ReplacementsYaml.h
new file mode 100644
index 00000000000..e19f4b43888
--- /dev/null
+++ b/clang-tools-extra/cpp11-migrate/Core/ReplacementsYaml.h
@@ -0,0 +1,102 @@
+//===-- Core/ReplacementsYaml.h ---------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief This file provides structs to store the migrator specific header
+/// replacements and the YAML traits for Replacements.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef CPP11_MIGRATE_REPLACEMENTS_YAML_H
+#define CPP11_MIGRATE_REPLACEMENTS_YAML_H
+
+#include "clang/Tooling/Refactoring.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <vector>
+#include <string>
+
+/// \brief The top-level YAML document that contains the details for the
+/// replacement.
+struct HeaderChangeDocument {
+  std::string TransformID;
+  std::vector<clang::tooling::Replacement> Replacements;
+  std::string HeaderFileName;
+  std::string SourceFileName;
+};
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(clang::tooling::Replacement)
+
+namespace llvm {
+namespace yaml {
+
+/// \brief ScalarTraits to read/write std::string objects.
+template <>
+struct ScalarTraits<std::string> {
+  static void output(const std::string &Val, void *, llvm::raw_ostream &Out) {
+    // We need to put quotes around the string to make sure special characters
+    // in the string is not treated as YAML tokens.
+    std::string NormalizedVal = std::string("\"") + Val + std::string("\"");
+    Out << NormalizedVal;
+  }
+
+  static StringRef input(StringRef Scalar, void *, std::string &Val) {
+    Val = Scalar;
+    return StringRef();
+  }
+};
+
+/// \brief Specialized MappingTraits for Repleacements to be converted to/from
+/// a YAML File.
+template <>
+struct MappingTraits<clang::tooling::Replacement> {
+  /// \brief Normalize clang::tooling::Replacement to provide direct access to
+  /// its members.
+  struct NormalizedReplacement {
+    NormalizedReplacement(const IO &)
+        : FilePath(""), Offset(0), Length(0), ReplacementText("") {}
+
+    NormalizedReplacement(const IO &, const clang::tooling::Replacement &R)
+        : FilePath(R.getFilePath()), Offset(R.getOffset()),
+          Length(R.getLength()), ReplacementText(R.getReplacementText()) {}
+
+    clang::tooling::Replacement denormalize(const IO &) {
+      return clang::tooling::Replacement(FilePath, Offset, Length,
+                                         ReplacementText);
+    }
+
+    std::string FilePath;
+    unsigned int Offset;
+    unsigned int Length;
+    std::string ReplacementText;
+  };
+
+  static void mapping(IO &Io, clang::tooling::Replacement &R) {
+    MappingNormalization<NormalizedReplacement, clang::tooling::Replacement>
+        Keys(Io, R);
+    Io.mapRequired("Offset", Keys->Offset);
+    Io.mapRequired("Length", Keys->Length);
+    Io.mapRequired("ReplacementText", Keys->ReplacementText);
+  }
+};
+
+/// \brief Specialized MappingTraits for HeaderChangeDocument to be converted
+/// to/from a YAML File.
+template <>
+struct MappingTraits<HeaderChangeDocument> {
+  static void mapping(IO &Io, HeaderChangeDocument &TD) {
+    Io.mapRequired("TransformID", TD.TransformID);
+    Io.mapRequired("Replacements", TD.Replacements);
+    Io.mapRequired("HeaderFileName", TD.HeaderFileName);
+    Io.mapRequired("SourceFileName", TD.SourceFileName);
+  }
+};
+} // end namespace yaml
+} // end namespace llvm
+
+#endif // CPP11_MIGRATE_REPLACEMENTS_YAML_H
diff --git a/clang-tools-extra/cpp11-migrate/Core/Transform.cpp b/clang-tools-extra/cpp11-migrate/Core/Transform.cpp
index 92520638502..bd08716d974 100644
--- a/clang-tools-extra/cpp11-migrate/Core/Transform.cpp
+++ b/clang-tools-extra/cpp11-migrate/Core/Transform.cpp
@@ -118,7 +118,7 @@ bool Transform::handleBeginSource(CompilerInstance &CI, StringRef Filename) {
 void Transform::handleEndSource() {
   if (!getReplacements().empty()) {
     SourceOverrides &SO = Overrides->getOrCreate(CurrentSource);
-    SO.applyReplacements(getReplacements());
+    SO.applyReplacements(getReplacements(), getName());
   }
 
   if (Options().EnableTiming)