[clangd] Construct SymbolSlab from YAML format.

Summary: This will be used together with D40548 for the global index source (experimental).

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: klimek, mgorny, ilya-biryukov, cfe-commits, ioeric

Differential Revision: https://reviews.llvm.org/D41178

llvm-svn: 320694

5 files changed, 210 insertions, 6 deletions

diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 29d6f46ae60..c4e0ebe9480 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -22,6 +22,7 @@ add_clang_library(clangDaemon
   index/MemIndex.cpp
   index/Index.cpp
   index/SymbolCollector.cpp
+  index/SymbolYAML.cpp
 
   LINK_LIBS
   clangAST
diff --git a/clang-tools-extra/clangd/index/Index.cpp b/clang-tools-extra/clangd/index/Index.cpp
index 79006bf27c5..806303cf05d 100644
--- a/clang-tools-extra/clangd/index/Index.cpp
+++ b/clang-tools-extra/clangd/index/Index.cpp
@@ -10,18 +10,24 @@
 #include "Index.h"
 
 #include "llvm/Support/SHA1.h"
+#include "llvm/ADT/StringExtras.h"
 
 namespace clang {
 namespace clangd {
 
-namespace {
-ArrayRef<uint8_t> toArrayRef(StringRef S) {
-  return {reinterpret_cast<const uint8_t *>(S.data()), S.size()};
+SymbolID::SymbolID(llvm::StringRef USR)
+    : HashValue(llvm::SHA1::hash(arrayRefFromStringRef(USR))) {}
+
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const SymbolID &ID) {
+  OS << toHex(llvm::toStringRef(ID.HashValue));
+  return OS;
 }
-} // namespace
 
-SymbolID::SymbolID(llvm::StringRef USR)
-    : HashValue(llvm::SHA1::hash(toArrayRef(USR))) {}
+void operator>>(llvm::StringRef Str, SymbolID &ID) {
+  std::string HexString = fromHex(Str);
+  assert(HexString.size() == 20);
+  std::copy(HexString.begin(), HexString.end(), ID.HashValue.begin());
+}
 
 SymbolSlab::const_iterator SymbolSlab::begin() const { return Symbols.begin(); }
 
diff --git a/clang-tools-extra/clangd/index/Index.h b/clang-tools-extra/clangd/index/Index.h
index c11749cbcd1..d36f720efcc 100644
--- a/clang-tools-extra/clangd/index/Index.h
+++ b/clang-tools-extra/clangd/index/Index.h
@@ -54,10 +54,22 @@ private:
   friend llvm::hash_code hash_value(const SymbolID &ID) {
     return hash_value(ArrayRef<uint8_t>(ID.HashValue));
   }
+  friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
+                                       const SymbolID &ID);
+  friend void operator>>(llvm::StringRef Str, SymbolID &ID);
 
   std::array<uint8_t, 20> HashValue;
 };
 
+// Write SymbolID into the given stream. SymbolID is encoded as a 40-bytes
+// hex string.
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const SymbolID &ID);
+
+// Construct SymbolID from a hex string.
+// The HexStr is required to be a 40-bytes hex string, which is encoded from the
+// "<<" operator.
+void operator>>(llvm::StringRef HexStr, SymbolID &ID);
+
 // The class presents a C++ symbol, e.g. class, function.
 //
 // FIXME: instead of having own copy fields for each symbol, we can share
diff --git a/clang-tools-extra/clangd/index/SymbolYAML.cpp b/clang-tools-extra/clangd/index/SymbolYAML.cpp
new file mode 100644
index 00000000000..bcd2985376b
--- /dev/null
+++ b/clang-tools-extra/clangd/index/SymbolYAML.cpp
@@ -0,0 +1,148 @@
+//===--- SymbolYAML.cpp ------------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolYAML.h"
+
+#include "Index.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Errc.h"
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(clang::clangd::Symbol)
+
+namespace llvm {
+namespace yaml {
+
+using clang::clangd::Symbol;
+using clang::clangd::SymbolID;
+using clang::clangd::SymbolLocation;
+using clang::index::SymbolInfo;
+using clang::index::SymbolLanguage;
+using clang::index::SymbolKind;
+
+// Helper to (de)serialize the SymbolID. We serialize it as a hex string.
+struct NormalizedSymbolID {
+  NormalizedSymbolID(IO &) {}
+  NormalizedSymbolID(IO &, const SymbolID& ID) {
+    llvm::raw_string_ostream OS(HexString);
+    OS << ID;
+  }
+
+  SymbolID denormalize(IO&) {
+    SymbolID ID;
+    HexString >> ID;
+    return ID;
+  }
+
+  std::string HexString;
+};
+
+template <> struct MappingTraits<SymbolLocation> {
+  static void mapping(IO &IO, SymbolLocation &Value) {
+    IO.mapRequired("StartOffset", Value.StartOffset);
+    IO.mapRequired("EndOffset", Value.EndOffset);
+    IO.mapRequired("FilePath", Value.FilePath);
+  }
+};
+
+template <> struct MappingTraits<SymbolInfo> {
+  static void mapping(IO &io, SymbolInfo &SymInfo) {
+    // FIXME: expose other fields?
+    io.mapRequired("Kind", SymInfo.Kind);
+    io.mapRequired("Lang", SymInfo.Lang);
+  }
+};
+
+template<> struct MappingTraits<Symbol> {
+  static void mapping(IO &IO, Symbol &Sym) {
+    MappingNormalization<NormalizedSymbolID, SymbolID> NSymbolID(
+        IO, Sym.ID);
+    IO.mapRequired("ID", NSymbolID->HexString);
+    IO.mapRequired("QualifiedName", Sym.QualifiedName);
+    IO.mapRequired("SymInfo", Sym.SymInfo);
+    IO.mapRequired("CanonicalDeclaration", Sym.CanonicalDeclaration);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolLanguage> {
+  static void enumeration(IO &IO, SymbolLanguage &Value) {
+    IO.enumCase(Value, "C", SymbolLanguage::C);
+    IO.enumCase(Value, "Cpp", SymbolLanguage::CXX);
+    IO.enumCase(Value, "ObjC", SymbolLanguage::ObjC);
+    IO.enumCase(Value, "Swift", SymbolLanguage::Swift);
+  }
+};
+
+template <> struct ScalarEnumerationTraits<SymbolKind> {
+  static void enumeration(IO &IO, SymbolKind &Value) {
+#define DEFINE_ENUM(name) IO.enumCase(Value, #name, SymbolKind::name)
+
+    DEFINE_ENUM(Unknown);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Module);
+    DEFINE_ENUM(Namespace);
+    DEFINE_ENUM(NamespaceAlias);
+    DEFINE_ENUM(Macro);
+    DEFINE_ENUM(Enum);
+    DEFINE_ENUM(Struct);
+    DEFINE_ENUM(Class);
+    DEFINE_ENUM(Protocol);
+    DEFINE_ENUM(Extension);
+    DEFINE_ENUM(Union);
+    DEFINE_ENUM(TypeAlias);
+    DEFINE_ENUM(Function);
+    DEFINE_ENUM(Variable);
+    DEFINE_ENUM(Field);
+    DEFINE_ENUM(EnumConstant);
+    DEFINE_ENUM(InstanceMethod);
+    DEFINE_ENUM(ClassMethod);
+    DEFINE_ENUM(StaticMethod);
+    DEFINE_ENUM(InstanceProperty);
+    DEFINE_ENUM(ClassProperty);
+    DEFINE_ENUM(StaticProperty);
+    DEFINE_ENUM(Constructor);
+    DEFINE_ENUM(Destructor);
+    DEFINE_ENUM(ConversionFunction);
+    DEFINE_ENUM(Parameter);
+    DEFINE_ENUM(Using);
+
+#undef DEFINE_ENUM
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+namespace clang {
+namespace clangd {
+
+SymbolSlab SymbolFromYAML(llvm::StringRef YAMLContent) {
+  std::vector<Symbol> S;
+  llvm::yaml::Input Yin(YAMLContent);
+  Yin >> S;
+  SymbolSlab Syms;
+  for (auto& Sym : S)
+    Syms.insert(std::move(Sym));
+  return Syms;
+}
+
+std::string SymbolToYAML(const SymbolSlab& Symbols) {
+  std::string Str;
+  llvm::raw_string_ostream OS(Str);
+  llvm::yaml::Output Yout(OS);
+  for (auto &Pair : Symbols) {
+    Symbol MutableSymbol = Pair.second;
+    Yout<< MutableSymbol;
+  }
+  return OS.str();
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/index/SymbolYAML.h b/clang-tools-extra/clangd/index/SymbolYAML.h
new file mode 100644
index 00000000000..49d0bf4f71d
--- /dev/null
+++ b/clang-tools-extra/clangd/index/SymbolYAML.h
@@ -0,0 +1,37 @@
+//===--- SymbolYAML.h --------------------------------------------*- C++-*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// SymbolYAML provides facilities to convert Symbol to YAML, and vice versa.
+// The YAML format of Symbol is designed for simplicity and experiment, but
+// isn't a suitable/efficient store.
+//
+// This is for **experimental** only. Don't use it in the production code.
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H
+
+#include "Index.h"
+#include "llvm/Support/Error.h"
+
+namespace clang {
+namespace clangd {
+
+// Read symbols from a YAML-format string.
+SymbolSlab SymbolFromYAML(llvm::StringRef YAMLContent);
+
+// Convert symbols to a YAML-format string.
+// The YAML result is safe to concatenate if you have multiple symbol slabs.
+std::string SymbolToYAML(const SymbolSlab& Symbols);
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_SYMBOL_FROM_YAML_H