[ORC] Re-enable the Error/Expected unit tests that were disabled in r300177.

The tests were failing due to an occasional deadlock in SerializationTraits
for Error: Both serializers and deserializers were protected by a single
mutex and in the unit test (where both ends of the RPC are in the same
process) one side might obtain the mutex, then block waiting for input,
leaving the other side of the connection unable to obtain the mutex to
write the data the first side was waiting for. Splitting the mutex into
two (one for serialization, one for deserialization) appears to have fixed the
issue.

llvm-svn: 300286

2 files changed, 166 insertions, 158 deletions

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h
index be4379e8867..84a037b2f99 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RPCSerialization.h
@@ -342,29 +342,35 @@ public:
     assert(!Name.empty() &&
            "The empty string is reserved for the Success value");
 
-    std::lock_guard<std::mutex> Lock(SerializersMutex);
-
-    // We're abusing the stability of std::map here: We take a reference to the
-    // key of the deserializers map to save us from duplicating the string in
-    // the serializer. This should be changed to use a stringpool if we switch
-    // to a map type that may move keys in memory.
-    auto I =
-      Deserializers.insert(Deserializers.begin(),
-                           std::make_pair(std::move(Name),
-                                          std::move(Deserialize)));
-
-    const std::string &KeyName = I->first;
-    // FIXME: Move capture Serialize once we have C++14.
-    Serializers[ErrorInfoT::classID()] =
-      [&KeyName, Serialize](ChannelT &C, const ErrorInfoBase &EIB) -> Error {
-        assert(EIB.dynamicClassID() == ErrorInfoT::classID() &&
-               "Serializer called for wrong error type");
-        if (auto Err = serializeSeq(C, KeyName))
-          return Err;
-        return Serialize(C, static_cast<const ErrorInfoT&>(EIB));
-      };
+    const std::string *KeyName = nullptr;
+    {
+      // We're abusing the stability of std::map here: We take a reference to the
+      // key of the deserializers map to save us from duplicating the string in
+      // the serializer. This should be changed to use a stringpool if we switch
+      // to a map type that may move keys in memory.
+      std::lock_guard<std::mutex> Lock(DeserializersMutex);
+      auto I =
+        Deserializers.insert(Deserializers.begin(),
+                             std::make_pair(std::move(Name),
+                                            std::move(Deserialize)));
+      KeyName = &I->first;
+    }
+    
+    {
+      assert(KeyName != nullptr && "No keyname pointer");
+      std::lock_guard<std::mutex> Lock(SerializersMutex);
+      // FIXME: Move capture Serialize once we have C++14.
+      Serializers[ErrorInfoT::classID()] =
+	[KeyName, Serialize](ChannelT &C, const ErrorInfoBase &EIB) -> Error {
+          assert(EIB.dynamicClassID() == ErrorInfoT::classID() &&
+		 "Serializer called for wrong error type");
+	  if (auto Err = serializeSeq(C, *KeyName))
+	    return Err;
+	  return Serialize(C, static_cast<const ErrorInfoT&>(EIB));
+        };
+    }
   }
-
+  
   static Error serialize(ChannelT &C, Error &&Err) {
     std::lock_guard<std::mutex> Lock(SerializersMutex);
     if (!Err)
@@ -380,7 +386,7 @@ public:
   }
 
   static Error deserialize(ChannelT &C, Error &Err) {
-    std::lock_guard<std::mutex> Lock(SerializersMutex);
+    std::lock_guard<std::mutex> Lock(DeserializersMutex);
 
     std::string Key;
     if (auto Err = deserializeSeq(C, Key))
@@ -412,6 +418,7 @@ private:
   }
 
   static std::mutex SerializersMutex;
+  static std::mutex DeserializersMutex;
   static std::map<const void*, WrappedErrorSerializer> Serializers;
   static std::map<std::string, WrappedErrorDeserializer> Deserializers;
 };
@@ -420,6 +427,9 @@ template <typename ChannelT>
 std::mutex SerializationTraits<ChannelT, Error>::SerializersMutex;
 
 template <typename ChannelT>
+std::mutex SerializationTraits<ChannelT, Error>::DeserializersMutex;
+
+template <typename ChannelT>
 std::map<const void*,
          typename SerializationTraits<ChannelT, Error>::WrappedErrorSerializer>
 SerializationTraits<ChannelT, Error>::Serializers;
diff --git a/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp b/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
index 8f6dddc62cd..1c9764b555f 100644
--- a/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
@@ -551,141 +551,139 @@ TEST(DummyRPC, TestWithAltCustomType) {
   ServerThread.join();
 }
 
-// FIXME: Temporarily disabled to investigate bot failure.
-
-// TEST(DummyRPC, ReturnErrorSuccess) {
-//   registerDummyErrorSerialization<QueueChannel>();
-
-//   auto Channels = createPairedQueueChannels();
-//   DummyRPCEndpoint Client(*Channels.first);
-//   DummyRPCEndpoint Server(*Channels.second);
-
-//   std::thread ServerThread([&]() {
-//       Server.addHandler<DummyRPCAPI::ErrorFunc>(
-//         []() {
-//           return Error::success();
-//         });
-
-//       // Handle the negotiate plus one call.
-//       for (unsigned I = 0; I != 2; ++I)
-//         cantFail(Server.handleOne());
-//     });
-
-//   cantFail(Client.callAsync<DummyRPCAPI::ErrorFunc>(
-//              [&](Error Err) {
-//                EXPECT_FALSE(!!Err) << "Expected success value";
-//                return Error::success();
-//              }));
-
-//   cantFail(Client.handleOne());
-
-//   ServerThread.join();
-// }
-
-// TEST(DummyRPC, ReturnErrorFailure) {
-//   registerDummyErrorSerialization<QueueChannel>();
-
-//   auto Channels = createPairedQueueChannels();
-//   DummyRPCEndpoint Client(*Channels.first);
-//   DummyRPCEndpoint Server(*Channels.second);
-
-//   std::thread ServerThread([&]() {
-//       Server.addHandler<DummyRPCAPI::ErrorFunc>(
-//         []() {
-//           return make_error<DummyError>(42);
-//         });
-
-//       // Handle the negotiate plus one call.
-//       for (unsigned I = 0; I != 2; ++I)
-//         cantFail(Server.handleOne());
-//     });
-
-//   cantFail(Client.callAsync<DummyRPCAPI::ErrorFunc>(
-//              [&](Error Err) {
-//                EXPECT_TRUE(Err.isA<DummyError>())
-//                  << "Incorrect error type";
-//                return handleErrors(
-//                         std::move(Err),
-//                         [](const DummyError &DE) {
-//                           EXPECT_EQ(DE.getValue(), 42ULL)
-//                             << "Incorrect DummyError serialization";
-//                         });
-//              }));
-
-//   cantFail(Client.handleOne());
-
-//   ServerThread.join();
-// }
-
-// TEST(DummyRPC, ReturnExpectedSuccess) {
-//   registerDummyErrorSerialization<QueueChannel>();
-
-//   auto Channels = createPairedQueueChannels();
-//   DummyRPCEndpoint Client(*Channels.first);
-//   DummyRPCEndpoint Server(*Channels.second);
-
-//   std::thread ServerThread([&]() {
-//       Server.addHandler<DummyRPCAPI::ExpectedFunc>(
-//         []() -> uint32_t {
-//           return 42;
-//         });
-
-//       // Handle the negotiate plus one call.
-//       for (unsigned I = 0; I != 2; ++I)
-//         cantFail(Server.handleOne());
-//     });
-
-//   cantFail(Client.callAsync<DummyRPCAPI::ExpectedFunc>(
-//                [&](Expected<uint32_t> ValOrErr) {
-//                  EXPECT_TRUE(!!ValOrErr)
-//                    << "Expected success value";
-//                  EXPECT_EQ(*ValOrErr, 42ULL)
-//                    << "Incorrect Expected<uint32_t> deserialization";
-//                  return Error::success();
-//                }));
-
-//   cantFail(Client.handleOne());
-
-//   ServerThread.join();
-// }
-
-// TEST(DummyRPC, ReturnExpectedFailure) {
-//   registerDummyErrorSerialization<QueueChannel>();
-
-//   auto Channels = createPairedQueueChannels();
-//   DummyRPCEndpoint Client(*Channels.first);
-//   DummyRPCEndpoint Server(*Channels.second);
-
-//   std::thread ServerThread([&]() {
-//       Server.addHandler<DummyRPCAPI::ExpectedFunc>(
-//         []() -> Expected<uint32_t> {
-//           return make_error<DummyError>(7);
-//         });
-
-//       // Handle the negotiate plus one call.
-//       for (unsigned I = 0; I != 2; ++I)
-//         cantFail(Server.handleOne());
-//     });
-
-//   cantFail(Client.callAsync<DummyRPCAPI::ExpectedFunc>(
-//                [&](Expected<uint32_t> ValOrErr) {
-//                  EXPECT_FALSE(!!ValOrErr)
-//                    << "Expected failure value";
-//                  auto Err = ValOrErr.takeError();
-//                  EXPECT_TRUE(Err.isA<DummyError>())
-//                    << "Incorrect error type";
-//                  return handleErrors(
-//                           std::move(Err),
-//                           [](const DummyError &DE) {
-//                             EXPECT_EQ(DE.getValue(), 7ULL)
-//                               << "Incorrect DummyError serialization";
-//                           });
-//                }));
-
-//   cantFail(Client.handleOne());
-
-//   ServerThread.join();
-// }
+TEST(DummyRPC, ReturnErrorSuccess) {
+  registerDummyErrorSerialization<QueueChannel>();
+
+  auto Channels = createPairedQueueChannels();
+  DummyRPCEndpoint Client(*Channels.first);
+  DummyRPCEndpoint Server(*Channels.second);
+
+  std::thread ServerThread([&]() {
+      Server.addHandler<DummyRPCAPI::ErrorFunc>(
+        []() {
+          return Error::success();
+        });
+
+      // Handle the negotiate plus one call.
+      for (unsigned I = 0; I != 2; ++I)
+        cantFail(Server.handleOne());
+    });
+
+  cantFail(Client.callAsync<DummyRPCAPI::ErrorFunc>(
+             [&](Error Err) {
+               EXPECT_FALSE(!!Err) << "Expected success value";
+               return Error::success();
+             }));
+
+  cantFail(Client.handleOne());
+
+  ServerThread.join();
+}
+
+TEST(DummyRPC, ReturnErrorFailure) {
+  registerDummyErrorSerialization<QueueChannel>();
+
+  auto Channels = createPairedQueueChannels();
+  DummyRPCEndpoint Client(*Channels.first);
+  DummyRPCEndpoint Server(*Channels.second);
+
+  std::thread ServerThread([&]() {
+      Server.addHandler<DummyRPCAPI::ErrorFunc>(
+        []() {
+          return make_error<DummyError>(42);
+        });
+
+      // Handle the negotiate plus one call.
+      for (unsigned I = 0; I != 2; ++I)
+        cantFail(Server.handleOne());
+    });
+
+  cantFail(Client.callAsync<DummyRPCAPI::ErrorFunc>(
+             [&](Error Err) {
+               EXPECT_TRUE(Err.isA<DummyError>())
+                 << "Incorrect error type";
+               return handleErrors(
+                        std::move(Err),
+                        [](const DummyError &DE) {
+                          EXPECT_EQ(DE.getValue(), 42ULL)
+                            << "Incorrect DummyError serialization";
+                        });
+             }));
+
+  cantFail(Client.handleOne());
+
+  ServerThread.join();
+}
+
+TEST(DummyRPC, ReturnExpectedSuccess) {
+  registerDummyErrorSerialization<QueueChannel>();
+
+  auto Channels = createPairedQueueChannels();
+  DummyRPCEndpoint Client(*Channels.first);
+  DummyRPCEndpoint Server(*Channels.second);
+
+  std::thread ServerThread([&]() {
+      Server.addHandler<DummyRPCAPI::ExpectedFunc>(
+        []() -> uint32_t {
+          return 42;
+        });
+
+      // Handle the negotiate plus one call.
+      for (unsigned I = 0; I != 2; ++I)
+        cantFail(Server.handleOne());
+    });
+
+  cantFail(Client.callAsync<DummyRPCAPI::ExpectedFunc>(
+               [&](Expected<uint32_t> ValOrErr) {
+                 EXPECT_TRUE(!!ValOrErr)
+                   << "Expected success value";
+                 EXPECT_EQ(*ValOrErr, 42ULL)
+                   << "Incorrect Expected<uint32_t> deserialization";
+                 return Error::success();
+               }));
+
+  cantFail(Client.handleOne());
+
+  ServerThread.join();
+}
+
+TEST(DummyRPC, ReturnExpectedFailure) {
+  registerDummyErrorSerialization<QueueChannel>();
+
+  auto Channels = createPairedQueueChannels();
+  DummyRPCEndpoint Client(*Channels.first);
+  DummyRPCEndpoint Server(*Channels.second);
+
+  std::thread ServerThread([&]() {
+      Server.addHandler<DummyRPCAPI::ExpectedFunc>(
+        []() -> Expected<uint32_t> {
+          return make_error<DummyError>(7);
+        });
+
+      // Handle the negotiate plus one call.
+      for (unsigned I = 0; I != 2; ++I)
+        cantFail(Server.handleOne());
+    });
+
+  cantFail(Client.callAsync<DummyRPCAPI::ExpectedFunc>(
+               [&](Expected<uint32_t> ValOrErr) {
+                 EXPECT_FALSE(!!ValOrErr)
+                   << "Expected failure value";
+                 auto Err = ValOrErr.takeError();
+                 EXPECT_TRUE(Err.isA<DummyError>())
+                   << "Incorrect error type";
+                 return handleErrors(
+                          std::move(Err),
+                          [](const DummyError &DE) {
+                            EXPECT_EQ(DE.getValue(), 7ULL)
+                              << "Incorrect DummyError serialization";
+                          });
+               }));
+
+  cantFail(Client.handleOne());
+
+  ServerThread.join();
+}
 
 TEST(DummyRPC, TestParallelCallGroup) {
   auto Channels = createPairedQueueChannels();