gdb-remote: Add jModulesInfo packet

Summary:
This adds the jModulesInfo packet, which is the equivalent of qModulesInfo, but it enables us to
query multiple modules at once. This makes a significant speed improvement in case the
application has many (over a hundred) modules, and the communication link has a non-negligible
latency. This functionality is accessed by ProcessGdbRemote::PrefetchModuleSpecs(), which does
the caching. GetModuleSpecs() is modified to first consult the cache before asking the remote
stub. PrefetchModuleSpecs is currently only called from POSIX-DYLD dynamic loader plugin, after
it reads the list of modules from the inferior memory, but other uses are possible.

This decreases the attach time to an android application by about 40%.

Reviewers: clayborg

Subscribers: tberghammer, lldb-commits, danalbert

Differential Revision: https://reviews.llvm.org/D24236

llvm-svn: 280919

6 files changed, 240 insertions, 17 deletions

diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index 6e09196ed21..68baba2494b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -32,6 +32,7 @@
 #include "lldb/Target/MemoryRegionInfo.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/UnixSignals.h"
+#include "lldb/Utility/JSON.h"
 #include "lldb/Utility/LLDBAssert.h"
 
 // Project includes
@@ -93,8 +94,8 @@ GDBRemoteCommunicationClient::GDBRemoteCommunicationClient()
       m_supports_z4(true), m_supports_QEnvironment(true),
       m_supports_QEnvironmentHexEncoded(true), m_supports_qSymbol(true),
       m_qSymbol_requests_done(false), m_supports_qModuleInfo(true),
-      m_supports_jThreadsInfo(true), m_curr_pid(LLDB_INVALID_PROCESS_ID),
-      m_curr_tid(LLDB_INVALID_THREAD_ID),
+      m_supports_jThreadsInfo(true), m_supports_jModulesInfo(true),
+      m_curr_pid(LLDB_INVALID_PROCESS_ID), m_curr_tid(LLDB_INVALID_THREAD_ID),
       m_curr_tid_run(LLDB_INVALID_THREAD_ID),
       m_num_supported_hardware_watchpoints(0), m_host_arch(), m_process_arch(),
       m_os_version_major(UINT32_MAX), m_os_version_minor(UINT32_MAX),
@@ -323,6 +324,7 @@ void GDBRemoteCommunicationClient::ResetDiscoverableSettings(bool did_exec) {
     m_qSupported_response.clear();
     m_supported_async_json_packets_is_valid = false;
     m_supported_async_json_packets_sp.reset();
+    m_supports_jModulesInfo = true;
   }
 
   // These flags should be reset when we first connect to a GDB server
@@ -3232,6 +3234,90 @@ bool GDBRemoteCommunicationClient::GetModuleInfo(
   return true;
 }
 
+static llvm::Optional<ModuleSpec>
+ParseModuleSpec(StructuredData::Dictionary *dict) {
+  ModuleSpec result;
+  if (!dict)
+    return llvm::None;
+
+  std::string string;
+  uint64_t integer;
+
+  if (!dict->GetValueForKeyAsString("uuid", string))
+    return llvm::None;
+  result.GetUUID().SetFromCString(string.c_str(), string.size());
+
+  if (!dict->GetValueForKeyAsInteger("file_offset", integer))
+    return llvm::None;
+  result.SetObjectOffset(integer);
+
+  if (!dict->GetValueForKeyAsInteger("file_size", integer))
+    return llvm::None;
+  result.SetObjectSize(integer);
+
+  if (!dict->GetValueForKeyAsString("triple", string))
+    return llvm::None;
+  result.GetArchitecture().SetTriple(string.c_str());
+
+  if (!dict->GetValueForKeyAsString("file_path", string))
+    return llvm::None;
+  result.GetFileSpec() = FileSpec(string, false, result.GetArchitecture());
+
+  return result;
+}
+
+llvm::Optional<std::vector<ModuleSpec>>
+GDBRemoteCommunicationClient::GetModulesInfo(
+    llvm::ArrayRef<FileSpec> module_file_specs, const llvm::Triple &triple) {
+  if (!m_supports_jModulesInfo)
+    return llvm::None;
+
+  JSONArray::SP module_array_sp = std::make_shared<JSONArray>();
+  for (const FileSpec &module_file_spec : module_file_specs) {
+    JSONObject::SP module_sp = std::make_shared<JSONObject>();
+    module_array_sp->AppendObject(module_sp);
+    module_sp->SetObject(
+        "file", std::make_shared<JSONString>(module_file_spec.GetPath()));
+    module_sp->SetObject("triple",
+                         std::make_shared<JSONString>(triple.getTriple()));
+  }
+  StreamString unescaped_payload;
+  unescaped_payload.PutCString("jModulesInfo:");
+  module_array_sp->Write(unescaped_payload);
+  StreamGDBRemote payload;
+  payload.PutEscapedBytes(unescaped_payload.GetData(),
+                          unescaped_payload.GetSize());
+
+  StringExtractorGDBRemote response;
+  if (SendPacketAndWaitForResponse(payload.GetString(), response, false) !=
+          PacketResult::Success ||
+      response.IsErrorResponse())
+    return llvm::None;
+
+  if (response.IsUnsupportedResponse()) {
+    m_supports_jModulesInfo = false;
+    return llvm::None;
+  }
+
+  StructuredData::ObjectSP response_object_sp =
+      StructuredData::ParseJSON(response.GetStringRef());
+  if (!response_object_sp)
+    return llvm::None;
+
+  StructuredData::Array *response_array = response_object_sp->GetAsArray();
+  if (!response_array)
+    return llvm::None;
+
+  std::vector<ModuleSpec> result;
+  for (size_t i = 0; i < response_array->GetSize(); ++i) {
+    if (llvm::Optional<ModuleSpec> module_spec = ParseModuleSpec(
+            response_array->GetItemAtIndex(i)->GetAsDictionary()))
+      result.push_back(*module_spec);
+  }
+
+  return result;
+}
+
 // query the target remote for extended information using the qXfer packet
 //
 // example: object='features', annex='target.xml', out=<xml output>
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
index e40c08cc681..cfdcc881d76 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
@@ -26,6 +26,8 @@
 #include "lldb/Core/StructuredData.h"
 #include "lldb/Target/Process.h"
 
+#include "llvm/ADT/Optional.h"
+
 namespace lldb_private {
 namespace process_gdb_remote {
 
@@ -437,6 +439,10 @@ public:
   bool GetModuleInfo(const FileSpec &module_file_spec,
                      const ArchSpec &arch_spec, ModuleSpec &module_spec);
 
+  llvm::Optional<std::vector<ModuleSpec>>
+  GetModulesInfo(llvm::ArrayRef<FileSpec> module_file_specs,
+                 const llvm::Triple &triple);
+
   bool ReadExtFeature(const lldb_private::ConstString object,
                       const lldb_private::ConstString annex, std::string &out,
                       lldb_private::Error &err);
@@ -527,7 +533,8 @@ protected:
       m_supports_z2 : 1, m_supports_z3 : 1, m_supports_z4 : 1,
       m_supports_QEnvironment : 1, m_supports_QEnvironmentHexEncoded : 1,
       m_supports_qSymbol : 1, m_qSymbol_requests_done : 1,
-      m_supports_qModuleInfo : 1, m_supports_jThreadsInfo : 1;
+      m_supports_qModuleInfo : 1, m_supports_jThreadsInfo : 1,
+      m_supports_jModulesInfo : 1;
 
   lldb::pid_t m_curr_pid;
   lldb::tid_t m_curr_tid; // Current gdb remote protocol thread index for all
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index f7ef2fe2176..0d7ca3c53e4 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -38,6 +38,7 @@
 #include "lldb/Target/FileAction.h"
 #include "lldb/Target/Platform.h"
 #include "lldb/Target/Process.h"
+#include "lldb/Utility/JSON.h"
 #include "llvm/ADT/Triple.h"
 
 // Project includes
@@ -102,6 +103,9 @@ GDBRemoteCommunicationServerCommon::GDBRemoteCommunicationServerCommon(
       StringExtractorGDBRemote::eServerPacketType_qModuleInfo,
       &GDBRemoteCommunicationServerCommon::Handle_qModuleInfo);
   RegisterMemberFunctionHandler(
+      StringExtractorGDBRemote::eServerPacketType_jModulesInfo,
+      &GDBRemoteCommunicationServerCommon::Handle_jModulesInfo);
+  RegisterMemberFunctionHandler(
       StringExtractorGDBRemote::eServerPacketType_qPlatform_chmod,
       &GDBRemoteCommunicationServerCommon::Handle_qPlatform_chmod);
   RegisterMemberFunctionHandler(
@@ -1078,22 +1082,11 @@ GDBRemoteCommunicationServerCommon::Handle_qModuleInfo(
 
   std::string triple;
   packet.GetHexByteString(triple);
-  ArchSpec arch(triple.c_str());
 
-  const FileSpec req_module_path_spec(module_path.c_str(), true);
-  const FileSpec module_path_spec =
-      FindModuleFile(req_module_path_spec.GetPath(), arch);
-  const ModuleSpec module_spec(module_path_spec, arch);
-
-  ModuleSpecList module_specs;
-  if (!ObjectFile::GetModuleSpecifications(module_path_spec, 0, 0,
-                                           module_specs))
+  ModuleSpec matched_module_spec = GetModuleInfo(module_path, triple);
+  if (!matched_module_spec.GetFileSpec())
     return SendErrorResponse(3);
 
-  ModuleSpec matched_module_spec;
-  if (!module_specs.FindMatchingModuleSpec(module_spec, matched_module_spec))
-    return SendErrorResponse(4);
-
   const auto file_offset = matched_module_spec.GetObjectOffset();
   const auto file_size = matched_module_spec.GetObjectSize();
   const auto uuid_str = matched_module_spec.GetUUID().GetAsString("");
@@ -1119,7 +1112,7 @@ GDBRemoteCommunicationServerCommon::Handle_qModuleInfo(
   response.PutChar(';');
 
   response.PutCString("file_path:");
-  response.PutCStringAsRawHex8(module_path_spec.GetCString());
+  response.PutCStringAsRawHex8(matched_module_spec.GetFileSpec().GetCString());
   response.PutChar(';');
   response.PutCString("file_offset:");
   response.PutHex64(file_offset);
@@ -1131,6 +1124,63 @@ GDBRemoteCommunicationServerCommon::Handle_qModuleInfo(
   return SendPacketNoLock(response.GetString());
 }
 
+GDBRemoteCommunication::PacketResult
+GDBRemoteCommunicationServerCommon::Handle_jModulesInfo(
+    StringExtractorGDBRemote &packet) {
+  packet.SetFilePos(::strlen("jModulesInfo:"));
+
+  StructuredData::ObjectSP object_sp = StructuredData::ParseJSON(packet.Peek());
+  if (!object_sp)
+    return SendErrorResponse(1);
+
+  StructuredData::Array *packet_array = object_sp->GetAsArray();
+  if (!packet_array)
+    return SendErrorResponse(2);
+
+  JSONArray::SP response_array_sp = std::make_shared<JSONArray>();
+  for (size_t i = 0; i < packet_array->GetSize(); ++i) {
+    StructuredData::Dictionary *query =
+        packet_array->GetItemAtIndex(i)->GetAsDictionary();
+    if (!query)
+      continue;
+    std::string file, triple;
+    if (!query->GetValueForKeyAsString("file", file) ||
+        !query->GetValueForKeyAsString("triple", triple))
+      continue;
+
+    ModuleSpec matched_module_spec = GetModuleInfo(file, triple);
+    if (!matched_module_spec.GetFileSpec())
+      continue;
+
+    const auto file_offset = matched_module_spec.GetObjectOffset();
+    const auto file_size = matched_module_spec.GetObjectSize();
+    const auto uuid_str = matched_module_spec.GetUUID().GetAsString("");
+
+    if (uuid_str.empty())
+      continue;
+
+    JSONObject::SP response = std::make_shared<JSONObject>();
+    response_array_sp->AppendObject(response);
+    response->SetObject("uuid", std::make_shared<JSONString>(uuid_str));
+    response->SetObject(
+        "triple",
+        std::make_shared<JSONString>(
+            matched_module_spec.GetArchitecture().GetTriple().getTriple()));
+    response->SetObject("file_path",
+                        std::make_shared<JSONString>(
+                            matched_module_spec.GetFileSpec().GetPath()));
+    response->SetObject("file_offset",
+                        std::make_shared<JSONNumber>(file_offset));
+    response->SetObject("file_size", std::make_shared<JSONNumber>(file_size));
+  }
+
+  StreamString response;
+  response_array_sp->Write(response);
+  StreamGDBRemote escaped_response;
+  escaped_response.PutEscapedBytes(response.GetData(), response.GetSize());
+  return SendPacketNoLock(escaped_response.GetString());
+}
+
 void GDBRemoteCommunicationServerCommon::CreateProcessInfoResponse(
     const ProcessInstanceInfo &proc_info, StreamString &response) {
   response.Printf(
@@ -1230,3 +1280,24 @@ FileSpec GDBRemoteCommunicationServerCommon::FindModuleFile(
   return FileSpec(module_path.c_str(), true);
 #endif
 }
+
+ModuleSpec GDBRemoteCommunicationServerCommon::GetModuleInfo(
+    const std::string &module_path, const std::string &triple) {
+  ArchSpec arch(triple.c_str());
+
+  const FileSpec req_module_path_spec(module_path.c_str(), true);
+  const FileSpec module_path_spec =
+      FindModuleFile(req_module_path_spec.GetPath(), arch);
+  const ModuleSpec module_spec(module_path_spec, arch);
+
+  ModuleSpecList module_specs;
+  if (!ObjectFile::GetModuleSpecifications(module_path_spec, 0, 0,
+                                           module_specs))
+    return ModuleSpec();
+
+  ModuleSpec matched_module_spec;
+  if (!module_specs.FindMatchingModuleSpec(module_spec, matched_module_spec))
+    return ModuleSpec();
+
+  return matched_module_spec;
+}
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.h
index d6be24a0e4b..321a92266bd 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.h
@@ -86,6 +86,8 @@ protected:
 
   PacketResult Handle_qModuleInfo(StringExtractorGDBRemote &packet);
 
+  PacketResult Handle_jModulesInfo(StringExtractorGDBRemote &packet);
+
   PacketResult Handle_qPlatform_shell(StringExtractorGDBRemote &packet);
 
   PacketResult Handle_qPlatform_mkdir(StringExtractorGDBRemote &packet);
@@ -149,6 +151,10 @@ protected:
 
   virtual FileSpec FindModuleFile(const std::string &module_path,
                                   const ArchSpec &arch);
+
+private:
+  ModuleSpec GetModuleInfo(const std::string &module_path,
+                           const std::string &triple);
 };
 
 } // namespace process_gdb_remote
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index 9e21e820dc7..cf220f3150b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -4020,6 +4020,14 @@ bool ProcessGDBRemote::GetModuleSpec(const FileSpec &module_file_spec,
                                      ModuleSpec &module_spec) {
   Log *log = GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PLATFORM);
 
+  const ModuleCacheKey key(module_file_spec.GetPath(),
+                           arch.GetTriple().getTriple());
+  auto cached = m_cached_module_specs.find(key);
+  if (cached != m_cached_module_specs.end()) {
+    module_spec = cached->second;
+    return bool(module_spec);
+  }
+
   if (!m_gdb_comm.GetModuleInfo(module_file_spec, arch, module_spec)) {
     if (log)
       log->Printf("ProcessGDBRemote::%s - failed to get module info for %s:%s",
@@ -4037,9 +4045,23 @@ bool ProcessGDBRemote::GetModuleSpec(const FileSpec &module_file_spec,
                 stream.GetString().c_str());
   }
 
+  m_cached_module_specs[key] = module_spec;
   return true;
 }
 
+void ProcessGDBRemote::PrefetchModuleSpecs(
+    llvm::ArrayRef<FileSpec> module_file_specs, const llvm::Triple &triple) {
+  auto module_specs = m_gdb_comm.GetModulesInfo(module_file_specs, triple);
+  if (module_specs) {
+    for (const FileSpec &spec : module_file_specs)
+      m_cached_module_specs[{spec.GetPath(), triple.getTriple()}] =
+          ModuleSpec();
+    for (const ModuleSpec &spec : *module_specs)
+      m_cached_module_specs[{spec.GetFileSpec().GetPath(),
+                             triple.getTriple()}] = spec;
+  }
+}
+
 bool ProcessGDBRemote::GetHostOSVersion(uint32_t &major, uint32_t &minor,
                                         uint32_t &update) {
   if (m_gdb_comm.GetOSVersion(major, minor, update))
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
index f2a58dbccaa..493176d72c2 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
@@ -25,6 +25,7 @@
 #include "lldb/Core/ConstString.h"
 #include "lldb/Core/Error.h"
 #include "lldb/Core/LoadedModuleInfoList.h"
+#include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/StreamString.h"
 #include "lldb/Core/StringList.h"
 #include "lldb/Core/StructuredData.h"
@@ -38,6 +39,8 @@
 #include "GDBRemoteCommunicationClient.h"
 #include "GDBRemoteRegisterContext.h"
 
+#include "llvm/ADT/DenseMap.h"
+
 namespace lldb_private {
 namespace process_gdb_remote {
 
@@ -194,6 +197,9 @@ public:
   bool GetModuleSpec(const FileSpec &module_file_spec, const ArchSpec &arch,
                      ModuleSpec &module_spec) override;
 
+  void PrefetchModuleSpecs(llvm::ArrayRef<FileSpec> module_file_specs,
+                           const llvm::Triple &triple) override;
+
   bool GetHostOSVersion(uint32_t &major, uint32_t &minor,
                         uint32_t &update) override;
 
@@ -412,6 +418,31 @@ private:
   bool
   HandleAsyncStructuredData(const StructuredData::ObjectSP &object_sp) override;
 
+  using ModuleCacheKey = std::pair<std::string, std::string>;
+  // KeyInfo for the cached module spec DenseMap.
+  // The invariant is that all real keys will have the file and architecture
+  // set.
+  // The empty key has an empty file and an empty arch.
+  // The tombstone key has an invalid arch and an empty file.
+  // The comparison and hash functions take the file name and architecture
+  // triple into account.
+  struct ModuleCacheInfo {
+    static ModuleCacheKey getEmptyKey() { return ModuleCacheKey(); }
+
+    static ModuleCacheKey getTombstoneKey() { return ModuleCacheKey("", "T"); }
+
+    static unsigned getHashValue(const ModuleCacheKey &key) {
+      return llvm::hash_combine(key.first, key.second);
+    }
+
+    static bool isEqual(const ModuleCacheKey &LHS, const ModuleCacheKey &RHS) {
+      return LHS == RHS;
+    }
+  };
+
+  llvm::DenseMap<ModuleCacheKey, ModuleSpec, ModuleCacheInfo>
+      m_cached_module_specs;
+
   DISALLOW_COPY_AND_ASSIGN(ProcessGDBRemote);
 };