[clangd] Store source file hash in IndexFile{In,Out}

Summary:
Puts the digest of the source file that generated the index into
serialized index and stores them back on load, if exists.

Reviewers: sammccall

Subscribers: ilya-biryukov, ioeric, MaskRay, jkorous, arphaman, cfe-commits

Differential Revision: https://reviews.llvm.org/D54693

llvm-svn: 347235

5 files changed, 51 insertions, 2 deletions

diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp
index bfb559f13fd..85544fa8502 100644
--- a/clang-tools-extra/clangd/index/Background.cpp
+++ b/clang-tools-extra/clangd/index/Background.cpp
@@ -246,6 +246,7 @@ void BackgroundIndex::update(StringRef MainFile, SymbolSlab Symbols,
       IndexFileOut Shard;
       Shard.Symbols = SS.get();
       Shard.Refs = RS.get();
+      Shard.Digest = &Hash;
       if (auto Error = IndexStorage->storeShard(Path, Shard))
         elog("Failed to write background-index shard for file {0}: {1}", Path,
              std::move(Error));
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index 7eea824ce0c..c1b42eadae0 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -363,6 +363,13 @@ Expected<IndexFileIn> readRIFF(StringRef Data) {
     return Strings.takeError();
 
   IndexFileIn Result;
+  if (Chunks.count("hash")) {
+    Reader Hash(Chunks.lookup("hash"));
+    llvm::StringRef Digest = Hash.consume(20);
+    Result.Digest.emplace();
+    std::copy(Digest.bytes_begin(), Digest.bytes_end(), Result.Digest->begin());
+  }
+
   if (Chunks.count("symb")) {
     Reader SymbolReader(Chunks.lookup("symb"));
     SymbolSlab::Builder Symbols;
@@ -399,6 +406,12 @@ void writeRIFF(const IndexFileOut &Data, raw_ostream &OS) {
   }
   RIFF.Chunks.push_back({riff::fourCC("meta"), Meta});
 
+  if (Data.Digest) {
+    llvm::StringRef Hash(reinterpret_cast<const char *>(Data.Digest->data()),
+                         Data.Digest->size());
+    RIFF.Chunks.push_back({riff::fourCC("hash"), Hash});
+  }
+
   StringTableOut Strings;
   std::vector<Symbol> Symbols;
   for (const auto &Sym : *Data.Symbols) {
diff --git a/clang-tools-extra/clangd/index/Serialization.h b/clang-tools-extra/clangd/index/Serialization.h
index 65b53ee3479..a2ee51a1e51 100644
--- a/clang-tools-extra/clangd/index/Serialization.h
+++ b/clang-tools-extra/clangd/index/Serialization.h
@@ -39,6 +39,8 @@ enum class IndexFileFormat {
 struct IndexFileIn {
   llvm::Optional<SymbolSlab> Symbols;
   llvm::Optional<RefSlab> Refs;
+  // Digest of the source file that generated the contents.
+  llvm::Optional<std::array<uint8_t, 20>> Digest;
 };
 // Parse an index file. The input must be a RIFF or YAML file.
 llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
@@ -47,6 +49,8 @@ llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef);
 struct IndexFileOut {
   const SymbolSlab *Symbols = nullptr;
   const RefSlab *Refs = nullptr;
+  // Digest of the source file that generated the contents.
+  const std::array<uint8_t, 20> *Digest = nullptr;
   // TODO: Support serializing Dex posting lists.
   IndexFileFormat Format = IndexFileFormat::RIFF;
 
diff --git a/clang-tools-extra/unittests/clangd/BackgroundIndexTests.cpp b/clang-tools-extra/unittests/clangd/BackgroundIndexTests.cpp
index 6746e800ea1..39f4040f09d 100644
--- a/clang-tools-extra/unittests/clangd/BackgroundIndexTests.cpp
+++ b/clang-tools-extra/unittests/clangd/BackgroundIndexTests.cpp
@@ -121,8 +121,10 @@ TEST(BackgroundIndexTest, ShardStorageWriteTest) {
       void f_b();
       class A_CC {};
       )cpp";
-  FS.Files[testPath("root/A.cc")] =
-      "#include \"A.h\"\nvoid g() { (void)common; }";
+  std::string A_CC = "#include \"A.h\"\nvoid g() { (void)common; }";
+  FS.Files[testPath("root/A.cc")] = A_CC;
+  auto Digest = llvm::SHA1::hash(
+      {reinterpret_cast<const uint8_t *>(A_CC.data()), A_CC.size()});
 
   llvm::StringMap<std::string> Storage;
   size_t CacheHits = 0;
@@ -156,6 +158,7 @@ TEST(BackgroundIndexTest, ShardStorageWriteTest) {
   EXPECT_NE(ShardSource, nullptr);
   EXPECT_THAT(*ShardSource->Symbols, UnorderedElementsAre());
   EXPECT_THAT(*ShardSource->Refs, RefsAre({FileURI("unittest:///root/A.cc")}));
+  EXPECT_EQ(*ShardSource->Digest, Digest);
 }
 
 } // namespace clangd
diff --git a/clang-tools-extra/unittests/clangd/SerializationTests.cpp b/clang-tools-extra/unittests/clangd/SerializationTests.cpp
index d3951228334..cf1bc428190 100644
--- a/clang-tools-extra/unittests/clangd/SerializationTests.cpp
+++ b/clang-tools-extra/unittests/clangd/SerializationTests.cpp
@@ -9,6 +9,7 @@
 
 #include "index/Index.h"
 #include "index/Serialization.h"
+#include "llvm/Support/SHA1.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
@@ -172,6 +173,33 @@ TEST(SerializationTest, BinaryConversions) {
               UnorderedElementsAreArray(YAMLFromRefs(*In->Refs)));
 }
 
+TEST(SerializationTest, HashTest) {
+  auto In = readIndexFile(YAML);
+  EXPECT_TRUE(bool(In)) << In.takeError();
+
+  std::string TestContent("TESTCONTENT");
+  auto Digest =
+      llvm::SHA1::hash({reinterpret_cast<const uint8_t *>(TestContent.data()),
+                        TestContent.size()});
+  // Write to binary format, and parse again.
+  IndexFileOut Out(*In);
+  Out.Format = IndexFileFormat::RIFF;
+  Out.Digest = &Digest;
+  std::string Serialized = to_string(Out);
+
+  auto In2 = readIndexFile(Serialized);
+  ASSERT_TRUE(bool(In2)) << In.takeError();
+  ASSERT_EQ(In2->Digest, Digest);
+  ASSERT_TRUE(In2->Symbols);
+  ASSERT_TRUE(In2->Refs);
+
+  // Assert the YAML serializations match, for nice comparisons and diffs.
+  EXPECT_THAT(YAMLFromSymbols(*In2->Symbols),
+              UnorderedElementsAreArray(YAMLFromSymbols(*In->Symbols)));
+  EXPECT_THAT(YAMLFromRefs(*In2->Refs),
+              UnorderedElementsAreArray(YAMLFromRefs(*In->Refs)));
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang