Don't call exit() from atexit handlers on Darwin

Summary:
Calling exit() from an atexit handler is undefined behavior.
On Linux, it's unavoidable, since we cannot intercept exit (_exit isn't called
if a user program uses return instead of exit()), and I haven't
seen it cause issues regardless.

However, on Darwin, I have a fairly complex internal test that hangs roughly
once in every 300 runs after leak reporting finishes, which is resolved with
this patch, and is presumably due to the undefined behavior (since the Die() is
the only thing that happens after the end of leak reporting).

In addition, this is the way TSan works as well, where an atexit handler+Die()
is used on Linux, and an _exit() interceptor is used on Darwin. I'm not sure if it's
intentionally structured that way in TSan, since TSan sets up the atexit handler and the
_exit() interceptor on both platforms, but I have observed that on Darwin, only the
_exit() interceptor is used, and on Linux the atexit handler is used.

There is some additional related discussion here: https://reviews.llvm.org/D35085

Reviewers: alekseyshl, kubamracek

Subscribers: eugenis, vsk, llvm-commits

Differential Revision: https://reviews.llvm.org/D35513

llvm-svn: 308353

6 files changed, 33 insertions, 7 deletions

diff --git a/compiler-rt/lib/asan/asan_interceptors.cc b/compiler-rt/lib/asan/asan_interceptors.cc
index ed12a9ac901..34ca22b8616 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cc
+++ b/compiler-rt/lib/asan/asan_interceptors.cc
@@ -178,6 +178,10 @@ void SetThreadName(const char *name) {
 }
 
 int OnExit() {
+  if (CAN_SANITIZE_LEAKS && common_flags()->detect_leaks &&
+      __lsan::HasReportedLeaks()) {
+    return common_flags()->exitcode;
+  }
   // FIXME: ask frontend whether we need to return failure.
   return 0;
 }
diff --git a/compiler-rt/lib/lsan/lsan_common.cc b/compiler-rt/lib/lsan/lsan_common.cc
index 4ffa91568cc..c121e6a8fb2 100644
--- a/compiler-rt/lib/lsan/lsan_common.cc
+++ b/compiler-rt/lib/lsan/lsan_common.cc
@@ -576,18 +576,16 @@ static bool CheckForLeaks() {
   return false;
 }
 
+static bool has_reported_leaks = false;
+bool HasReportedLeaks() { return has_reported_leaks; }
+
 void DoLeakCheck() {
   BlockingMutexLock l(&global_mutex);
   static bool already_done;
   if (already_done) return;
   already_done = true;
-  bool have_leaks = CheckForLeaks();
-  if (!have_leaks) {
-    return;
-  }
-  if (common_flags()->exitcode) {
-    Die();
-  }
+  has_reported_leaks = CheckForLeaks();
+  if (has_reported_leaks) HandleLeaks();
 }
 
 static int DoRecoverableLeakCheck() {
diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h
index d93ac1b1091..31bf3eb1df4 100644
--- a/compiler-rt/lib/lsan/lsan_common.h
+++ b/compiler-rt/lib/lsan/lsan_common.h
@@ -226,6 +226,12 @@ IgnoreObjectResult IgnoreObjectLocked(const void *p);
 // Return the linker module, if valid for the platform.
 LoadedModule *GetLinker();
 
+// Return true if LSan has finished leak checking and reported leaks.
+bool HasReportedLeaks();
+
+// Run platform-specific leak handlers.
+void HandleLeaks();
+
 // Wrapper for chunk metadata operations.
 class LsanMetadata {
  public:
diff --git a/compiler-rt/lib/lsan/lsan_common_linux.cc b/compiler-rt/lib/lsan/lsan_common_linux.cc
index c903be42d1e..5042c7b3ada 100644
--- a/compiler-rt/lib/lsan/lsan_common_linux.cc
+++ b/compiler-rt/lib/lsan/lsan_common_linux.cc
@@ -100,6 +100,13 @@ struct DoStopTheWorldParam {
   void *argument;
 };
 
+// While calling Die() here is undefined behavior and can potentially
+// cause race conditions, it isn't possible to intercept exit on linux,
+// so we have no choice but to call Die() from the atexit handler.
+void HandleLeaks() {
+  if (common_flags()->exitcode) Die();
+}
+
 static int DoStopTheWorldCallback(struct dl_phdr_info *info, size_t size,
                                   void *data) {
   DoStopTheWorldParam *param = reinterpret_cast<DoStopTheWorldParam *>(data);
diff --git a/compiler-rt/lib/lsan/lsan_common_mac.cc b/compiler-rt/lib/lsan/lsan_common_mac.cc
index c0fa36fd969..147e62d1fe8 100644
--- a/compiler-rt/lib/lsan/lsan_common_mac.cc
+++ b/compiler-rt/lib/lsan/lsan_common_mac.cc
@@ -171,6 +171,11 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
   }
 }
 
+// On darwin, we can intercept _exit gracefully, and return a failing exit code
+// if required at that point. Calling Die() here is undefined behavior and
+// causes rare race conditions.
+void HandleLeaks() {}
+
 void DoStopTheWorld(StopTheWorldCallback callback, void *argument) {
   StopTheWorld(callback, argument);
 }
diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cc b/compiler-rt/lib/lsan/lsan_interceptors.cc
index 7d514402ad4..168868b012b 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cc
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cc
@@ -352,6 +352,11 @@ INTERCEPTOR(int, pthread_join, void *th, void **ret) {
   return res;
 }
 
+INTERCEPTOR(void, _exit, int status) {
+  if (status == 0 && HasReportedLeaks()) status = common_flags()->exitcode;
+  REAL(_exit)(status);
+}
+
 namespace __lsan {
 
 void InitializeInterceptors() {
@@ -371,6 +376,7 @@ void InitializeInterceptors() {
   LSAN_MAYBE_INTERCEPT_MALLOPT;
   INTERCEPT_FUNCTION(pthread_create);
   INTERCEPT_FUNCTION(pthread_join);
+  INTERCEPT_FUNCTION(_exit);
 
   if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
     Report("LeakSanitizer: failed to create thread key.\n");