[clang-doc] Implement a YAML generator

Implmenting a YAML generator from the emitted bitcode summary of
declarations. Emits one YAML file for each declaration information.

For a more detailed overview of the tool, see the design document on the mailing list: http://lists.llvm.org/pipermail/cfe-dev/2017-December/056203.html

llvm-svn: 334103

6 files changed, 428 insertions, 16 deletions

diff --git a/clang-tools-extra/clang-doc/CMakeLists.txt b/clang-tools-extra/clang-doc/CMakeLists.txt
index 97933a3dade..eaebf616f6f 100644
--- a/clang-tools-extra/clang-doc/CMakeLists.txt
+++ b/clang-tools-extra/clang-doc/CMakeLists.txt
@@ -8,9 +8,11 @@ add_clang_library(clangDoc
   BitcodeReader.cpp
   BitcodeWriter.cpp
   ClangDoc.cpp
+  Generators.cpp
   Mapper.cpp
   Representation.cpp
   Serialize.cpp
+  YAMLGenerator.cpp
 
   LINK_LIBS
   clangAnalysis
diff --git a/clang-tools-extra/clang-doc/Generators.cpp b/clang-tools-extra/clang-doc/Generators.cpp
new file mode 100644
index 00000000000..fe01d610970
--- /dev/null
+++ b/clang-tools-extra/clang-doc/Generators.cpp
@@ -0,0 +1,36 @@
+//===---- Generator.cpp - Generator Registry ---------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Generators.h"
+
+LLVM_INSTANTIATE_REGISTRY(clang::doc::GeneratorRegistry)
+
+namespace clang {
+namespace doc {
+
+llvm::Expected<std::unique_ptr<Generator>>
+findGeneratorByName(llvm::StringRef Format) {
+  for (auto I = GeneratorRegistry::begin(), E = GeneratorRegistry::end();
+       I != E; ++I) {
+    if (I->getName() != Format)
+      continue;
+    return I->instantiate();
+  }
+  return llvm::make_error<llvm::StringError>("Can't find generator: " + Format,
+                                             llvm::inconvertibleErrorCode());
+}
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the generators.
+extern volatile int YAMLGeneratorAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED YAMLGeneratorAnchorDest =
+    YAMLGeneratorAnchorSource;
+
+} // namespace doc
+} // namespace clang
diff --git a/clang-tools-extra/clang-doc/Generators.h b/clang-tools-extra/clang-doc/Generators.h
new file mode 100644
index 00000000000..9106d2cff6a
--- /dev/null
+++ b/clang-tools-extra/clang-doc/Generators.h
@@ -0,0 +1,41 @@
+//===-- Generators.h - ClangDoc Generator ----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Generator classes for converting declaration information into documentation
+// in a specified format.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_GENERATOR_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_GENERATOR_H
+
+#include "Representation.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/Registry.h"
+
+namespace clang {
+namespace doc {
+
+// Abstract base class for generators.
+// This is expected to be implemented and exposed via the GeneratorRegistry.
+class Generator {
+public:
+  virtual ~Generator() = default;
+
+  // Write out the decl info in the specified format.
+  virtual bool generateDocForInfo(Info *I, llvm::raw_ostream &OS) = 0;
+};
+
+typedef llvm::Registry<Generator> GeneratorRegistry;
+
+llvm::Expected<std::unique_ptr<Generator>>
+findGeneratorByName(llvm::StringRef Format);
+
+} // namespace doc
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_DOC_GENERATOR_H
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index 28968754732..8b3a2c117ba 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -73,6 +73,11 @@ struct Reference {
   Reference(SymbolID USR, StringRef Name, InfoType IT)
       : USR(USR), Name(Name), RefType(IT) {}
 
+  bool operator==(const Reference &Other) const {
+    return std::tie(USR, Name, RefType) ==
+           std::tie(Other.USR, Other.Name, Other.RefType);
+  }
+
   SymbolID USR = SymbolID(); // Unique identifer for referenced decl
   SmallString<16> Name;      // Name of type (possibly unresolved).
   InfoType RefType = InfoType::IT_default; // Indicates the type of this
@@ -87,6 +92,8 @@ struct TypeInfo {
       : Type(Type, Field, IT) {}
   TypeInfo(llvm::StringRef RefName) : Type(RefName) {}
 
+  bool operator==(const TypeInfo &Other) const { return Type == Other.Type; }
+
   Reference Type; // Referenced type in this info.
 };
 
@@ -99,6 +106,10 @@ struct FieldTypeInfo : public TypeInfo {
   FieldTypeInfo(llvm::StringRef RefName, llvm::StringRef Name)
       : TypeInfo(RefName), Name(Name) {}
 
+  bool operator==(const FieldTypeInfo &Other) const {
+    return std::tie(Type, Name) == std::tie(Other.Type, Other.Name);
+  }
+
   SmallString<16> Name; // Name associated with this info.
 };
 
@@ -112,6 +123,11 @@ struct MemberTypeInfo : public FieldTypeInfo {
                  AccessSpecifier Access)
       : FieldTypeInfo(RefName, Name), Access(Access) {}
 
+  bool operator==(const MemberTypeInfo &Other) const {
+    return std::tie(Type, Name, Access) ==
+           std::tie(Other.Type, Other.Name, Other.Access);
+  }
+
   AccessSpecifier Access = AccessSpecifier::AS_none; // Access level associated
                                                      // with this info (public,
                                                      // protected, private,
@@ -123,6 +139,11 @@ struct Location {
   Location(int LineNumber, SmallString<16> Filename)
       : LineNumber(LineNumber), Filename(std::move(Filename)) {}
 
+  bool operator==(const Location &Other) const {
+    return std::tie(LineNumber, Filename) ==
+           std::tie(Other.LineNumber, Other.Filename);
+  }
+
   int LineNumber;           // Line number of this Location.
   SmallString<32> Filename; // File for this Location.
 };
@@ -173,10 +194,9 @@ struct FunctionInfo : public SymbolInfo {
   bool IsMethod = false; // Indicates whether this function is a class method.
   Reference Parent;      // Reference to the parent class decl for this method.
   TypeInfo ReturnType;   // Info about the return type of this function.
-  llvm::SmallVector<FieldTypeInfo, 4> Params;        // List of parameters.
-  AccessSpecifier Access = AccessSpecifier::AS_none; // Access level for this
-                                                     // method (public, private,
-                                                     // protected, none).
+  llvm::SmallVector<FieldTypeInfo, 4> Params; // List of parameters.
+  // Access level for this method (public, private, protected, none).
+  AccessSpecifier Access = AccessSpecifier::AS_none;
 };
 
 // TODO: Expand to allow for documenting templating, inheritance access,
diff --git a/clang-tools-extra/clang-doc/YAMLGenerator.cpp b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
new file mode 100644
index 00000000000..f29b4787dbd
--- /dev/null
+++ b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
@@ -0,0 +1,268 @@
+//===--  ClangDocYAML.cpp - ClangDoc YAML -----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+// Implementation of the YAML generator, converting decl info into YAML output.
+//===----------------------------------------------------------------------===//
+
+#include "Generators.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang::doc;
+
+LLVM_YAML_IS_SEQUENCE_VECTOR(FieldTypeInfo)
+LLVM_YAML_IS_SEQUENCE_VECTOR(MemberTypeInfo)
+LLVM_YAML_IS_SEQUENCE_VECTOR(Reference)
+LLVM_YAML_IS_SEQUENCE_VECTOR(Location)
+LLVM_YAML_IS_SEQUENCE_VECTOR(CommentInfo)
+LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<CommentInfo>)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::SmallString<16>)
+
+namespace llvm {
+namespace yaml {
+
+// Enumerations to YAML output.
+
+template <> struct ScalarEnumerationTraits<clang::AccessSpecifier> {
+  static void enumeration(IO &IO, clang::AccessSpecifier &Value) {
+    IO.enumCase(Value, "Public", clang::AccessSpecifier::AS_public);
+    IO.enumCase(Value, "Protected", clang::AccessSpecifier::AS_protected);
+    IO.enumCase(Value, "Private", clang::AccessSpecifier::AS_private);
+    IO.enumCase(Value, "None", clang::AccessSpecifier::AS_none);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<clang::TagTypeKind> {
+  static void enumeration(IO &IO, clang::TagTypeKind &Value) {
+    IO.enumCase(Value, "Struct", clang::TagTypeKind::TTK_Struct);
+    IO.enumCase(Value, "Interface", clang::TagTypeKind::TTK_Interface);
+    IO.enumCase(Value, "Union", clang::TagTypeKind::TTK_Union);
+    IO.enumCase(Value, "Class", clang::TagTypeKind::TTK_Class);
+    IO.enumCase(Value, "Enum", clang::TagTypeKind::TTK_Enum);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<InfoType> {
+  static void enumeration(IO &IO, InfoType &Value) {
+    IO.enumCase(Value, "Namespace", InfoType::IT_namespace);
+    IO.enumCase(Value, "Record", InfoType::IT_record);
+    IO.enumCase(Value, "Function", InfoType::IT_function);
+    IO.enumCase(Value, "Enum", InfoType::IT_enum);
+    IO.enumCase(Value, "Default", InfoType::IT_default);
+  }
+};
+
+// Scalars to YAML output.
+template <unsigned U> struct ScalarTraits<SmallString<U>> {
+
+  static void output(const SmallString<U> &S, void *, llvm::raw_ostream &OS) {
+    for (const auto &C : S)
+      OS << C;
+  }
+
+  static StringRef input(StringRef Scalar, void *, SmallString<U> &Value) {
+    Value.assign(Scalar.begin(), Scalar.end());
+    return StringRef();
+  }
+
+  static QuotingType mustQuote(StringRef) { return QuotingType::Single; }
+};
+
+template <> struct ScalarTraits<std::array<unsigned char, 20>> {
+
+  static void output(const std::array<unsigned char, 20> &S, void *,
+                     llvm::raw_ostream &OS) {
+    OS << toHex(toStringRef(S));
+  }
+
+  static StringRef input(StringRef Scalar, void *,
+                         std::array<unsigned char, 20> &Value) {
+    if (Scalar.size() != 40)
+      return "Error: Incorrect scalar size for USR.";
+    Value = StringToSymbol(Scalar);
+    return StringRef();
+  }
+
+  static SymbolID StringToSymbol(llvm::StringRef Value) {
+    SymbolID USR;
+    std::string HexString = fromHex(Value);
+    std::copy(HexString.begin(), HexString.end(), USR.begin());
+    return SymbolID(USR);
+  }
+
+  static QuotingType mustQuote(StringRef) { return QuotingType::Single; }
+};
+
+// Helper functions to map infos to YAML.
+
+static void TypeInfoMapping(IO &IO, TypeInfo &I) {
+  IO.mapOptional("Type", I.Type, Reference());
+}
+
+static void FieldTypeInfoMapping(IO &IO, FieldTypeInfo &I) {
+  TypeInfoMapping(IO, I);
+  IO.mapOptional("Name", I.Name, SmallString<16>());
+}
+
+static void InfoMapping(IO &IO, Info &I) {
+  IO.mapRequired("USR", I.USR);
+  IO.mapOptional("Name", I.Name, SmallString<16>());
+  IO.mapOptional("Namespace", I.Namespace, llvm::SmallVector<Reference, 4>());
+  IO.mapOptional("Description", I.Description);
+}
+
+static void SymbolInfoMapping(IO &IO, SymbolInfo &I) {
+  InfoMapping(IO, I);
+  IO.mapOptional("DefLocation", I.DefLoc, Optional<Location>());
+  IO.mapOptional("Location", I.Loc, llvm::SmallVector<Location, 2>());
+}
+
+static void CommentInfoMapping(IO &IO, CommentInfo &I) {
+  IO.mapOptional("Kind", I.Kind, SmallString<16>());
+  IO.mapOptional("Text", I.Text, SmallString<64>());
+  IO.mapOptional("Name", I.Name, SmallString<16>());
+  IO.mapOptional("Direction", I.Direction, SmallString<8>());
+  IO.mapOptional("ParamName", I.ParamName, SmallString<16>());
+  IO.mapOptional("CloseName", I.CloseName, SmallString<16>());
+  IO.mapOptional("SelfClosing", I.SelfClosing, false);
+  IO.mapOptional("Explicit", I.Explicit, false);
+  IO.mapOptional("Args", I.Args, llvm::SmallVector<SmallString<16>, 4>());
+  IO.mapOptional("AttrKeys", I.AttrKeys,
+                 llvm::SmallVector<SmallString<16>, 4>());
+  IO.mapOptional("AttrValues", I.AttrValues,
+                 llvm::SmallVector<SmallString<16>, 4>());
+  IO.mapOptional("Children", I.Children);
+}
+
+// Template specialization to YAML traits for Infos.
+
+template <> struct MappingTraits<Location> {
+  static void mapping(IO &IO, Location &Loc) {
+    IO.mapOptional("LineNumber", Loc.LineNumber, 0);
+    IO.mapOptional("Filename", Loc.Filename, SmallString<32>());
+  }
+};
+
+template <> struct MappingTraits<Reference> {
+  static void mapping(IO &IO, Reference &Ref) {
+    IO.mapOptional("Type", Ref.RefType, InfoType::IT_default);
+    IO.mapOptional("Name", Ref.Name, SmallString<16>());
+    IO.mapOptional("USR", Ref.USR, SymbolID());
+  }
+};
+
+template <> struct MappingTraits<TypeInfo> {
+  static void mapping(IO &IO, TypeInfo &I) { TypeInfoMapping(IO, I); }
+};
+
+template <> struct MappingTraits<FieldTypeInfo> {
+  static void mapping(IO &IO, FieldTypeInfo &I) {
+    TypeInfoMapping(IO, I);
+    IO.mapOptional("Name", I.Name, SmallString<16>());
+  }
+};
+
+template <> struct MappingTraits<MemberTypeInfo> {
+  static void mapping(IO &IO, MemberTypeInfo &I) {
+    FieldTypeInfoMapping(IO, I);
+    IO.mapOptional("Access", I.Access, clang::AccessSpecifier::AS_none);
+  }
+};
+
+template <> struct MappingTraits<NamespaceInfo> {
+  static void mapping(IO &IO, NamespaceInfo &I) { InfoMapping(IO, I); }
+};
+
+template <> struct MappingTraits<RecordInfo> {
+  static void mapping(IO &IO, RecordInfo &I) {
+    SymbolInfoMapping(IO, I);
+    IO.mapOptional("TagType", I.TagType, clang::TagTypeKind::TTK_Struct);
+    IO.mapOptional("Members", I.Members);
+    IO.mapOptional("Parents", I.Parents, llvm::SmallVector<Reference, 4>());
+    IO.mapOptional("VirtualParents", I.VirtualParents,
+                   llvm::SmallVector<Reference, 4>());
+  }
+};
+
+template <> struct MappingTraits<EnumInfo> {
+  static void mapping(IO &IO, EnumInfo &I) {
+    SymbolInfoMapping(IO, I);
+    IO.mapOptional("Scoped", I.Scoped, false);
+    IO.mapOptional("Members", I.Members);
+  }
+};
+
+template <> struct MappingTraits<FunctionInfo> {
+  static void mapping(IO &IO, FunctionInfo &I) {
+    SymbolInfoMapping(IO, I);
+    IO.mapOptional("IsMethod", I.IsMethod, false);
+    IO.mapOptional("Parent", I.Parent, Reference());
+    IO.mapOptional("Params", I.Params);
+    IO.mapOptional("ReturnType", I.ReturnType);
+    IO.mapOptional("Access", I.Access, clang::AccessSpecifier::AS_none);
+  }
+};
+
+template <> struct MappingTraits<CommentInfo> {
+  static void mapping(IO &IO, CommentInfo &I) { CommentInfoMapping(IO, I); }
+};
+
+template <> struct MappingTraits<std::unique_ptr<CommentInfo>> {
+  static void mapping(IO &IO, std::unique_ptr<CommentInfo> &I) {
+    if (I)
+      CommentInfoMapping(IO, *I);
+  }
+};
+
+} // end namespace yaml
+} // end namespace llvm
+
+namespace clang {
+namespace doc {
+
+/// Generator for YAML documentation.
+class YAMLGenerator : public Generator {
+public:
+  static const char *Format;
+
+  bool generateDocForInfo(Info *I, llvm::raw_ostream &OS) override;
+};
+
+const char *YAMLGenerator::Format = "yaml";
+
+bool YAMLGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS) {
+  llvm::yaml::Output InfoYAML(OS);
+  switch (I->IT) {
+  case InfoType::IT_namespace:
+    InfoYAML << *static_cast<clang::doc::NamespaceInfo *>(I);
+    break;
+  case InfoType::IT_record:
+    InfoYAML << *static_cast<clang::doc::RecordInfo *>(I);
+    break;
+  case InfoType::IT_enum:
+    InfoYAML << *static_cast<clang::doc::EnumInfo *>(I);
+    break;
+  case InfoType::IT_function:
+    InfoYAML << *static_cast<clang::doc::FunctionInfo *>(I);
+    break;
+  case InfoType::IT_default:
+    llvm::errs() << "Unexpected info type in index.\n";
+    return true;
+  }
+  return false;
+}
+
+static GeneratorRegistry::Add<YAMLGenerator> YAML(YAMLGenerator::Format,
+                                                  "Generator for YAML output.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the generator.
+volatile int YAMLGeneratorAnchorSource = 0;
+
+} // namespace doc
+} // namespace clang
diff --git a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
index 759342943e7..9345dad87c0 100644
--- a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -21,6 +21,7 @@
 #include "BitcodeReader.h"
 #include "BitcodeWriter.h"
 #include "ClangDoc.h"
+#include "Generators.h"
 #include "Representation.h"
 #include "clang/AST/AST.h"
 #include "clang/AST/Decl.h"
@@ -67,10 +68,10 @@ enum OutputFormatTy {
   yaml,
 };
 
-static llvm::cl::opt<OutputFormatTy>
-    Format("format", llvm::cl::desc("Format for outputted docs."),
-           llvm::cl::values(clEnumVal(yaml, "Documentation in YAML format.")),
-           llvm::cl::init(yaml), llvm::cl::cat(ClangDocCategory));
+static llvm::cl::opt<OutputFormatTy> FormatEnum(
+    "format", llvm::cl::desc("Format for outputted docs."),
+    llvm::cl::values(clEnumVal(yaml, "Documentation in YAML format.")),
+    llvm::cl::init(yaml), llvm::cl::cat(ClangDocCategory));
 
 static llvm::cl::opt<bool> DoxygenOnly(
     "doxygen",
@@ -116,8 +117,41 @@ bool DumpResultToFile(const Twine &DirName, const Twine &FileName,
   return false;
 }
 
+llvm::Expected<llvm::SmallString<128>>
+getPath(StringRef Root, StringRef Ext, StringRef Name,
+        llvm::SmallVectorImpl<doc::Reference> &Namespaces) {
+  std::error_code OK;
+  llvm::SmallString<128> Path;
+  llvm::sys::path::native(Root, Path);
+  for (auto R = Namespaces.rbegin(), E = Namespaces.rend(); R != E; ++R)
+    llvm::sys::path::append(Path, R->Name);
+
+  if (CreateDirectory(Path))
+    return llvm::make_error<llvm::StringError>("Unable to create directory.\n",
+                                               llvm::inconvertibleErrorCode());
+
+  llvm::sys::path::append(Path, Name + Ext);
+  return Path;
+}
+
+std::string getFormatString(OutputFormatTy Ty) {
+  switch (Ty) {
+  case yaml:
+    return "yaml";
+  }
+}
+
 int main(int argc, const char **argv) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
+  std::error_code OK;
+
+  // Fail early if an invalid format was provided.
+  std::string Format = getFormatString(FormatEnum);
+  auto G = doc::findGeneratorByName(Format);
+  if (!G) {
+    llvm::errs() << toString(G.takeError()) << "\n";
+    return 1;
+  }
 
   auto Exec = clang::tooling::createExecutorFromCommandLineArgs(
       argc, argv, ClangDocCategory);
@@ -173,7 +207,7 @@ int main(int argc, const char **argv) {
     }
   });
 
-  // Reducing phase
+  // Reducing and generation phases
   llvm::outs() << "Reducing " << MapOutput.size() << " infos...\n";
   llvm::StringMap<std::unique_ptr<doc::Info>> ReduceOutput;
   for (auto &Group : MapOutput) {
@@ -186,16 +220,27 @@ int main(int argc, const char **argv) {
       llvm::BitstreamWriter Stream(Buffer);
       doc::ClangDocBitcodeWriter Writer(Stream);
       Writer.dispatchInfoForWrite(Reduced.get().get());
-      if (DumpResultToFile("bc", Group.getKey() + ".bc", Buffer)) {
-        llvm::errs() << "Error writing " << Group.getKey() << " to file.\n";
-        continue;
-      }
+      if (DumpResultToFile("bc", Group.getKey() + ".bc", Buffer))
+        llvm::errs() << "Error dumping to bitcode.\n";
+      continue;
     }
 
-    ReduceOutput.insert(
-        std::make_pair(Group.getKey(), std::move(Reduced.get())));
+    // Create the relevant ostream and emit the documentation for this decl.
+    doc::Info *I = Reduced.get().get();
+    auto InfoPath = getPath(OutDirectory, "." + Format, I->Name, I->Namespace);
+    if (!InfoPath) {
+      llvm::errs() << toString(InfoPath.takeError()) << "\n";
+      continue;
+    }
+    std::error_code FileErr;
+    llvm::raw_fd_ostream InfoOS(InfoPath.get(), FileErr, llvm::sys::fs::F_None);
+    if (FileErr != OK) {
+      llvm::errs() << "Error opening index file: " << FileErr.message() << "\n";
+      continue;
+    }
 
-    // FIXME: Add support for emitting different output formats.
+    if (G->get()->generateDocForInfo(I, InfoOS))
+      llvm::errs() << "Unable to generate docs for info.\n";
   }
 
   return 0;