Introduce a way to allow the ASan dylib on Darwin platforms to be loaded via `dlopen()`.

Summary:

The purpose of this option is provide a way for the ASan dylib
to be loaded via `dlopen()` without triggering most initialization
steps (e.g. shadow memory set up) that normally occur when the
ASan dylib is loaded.

This new functionality is exposed by

- A `SANITIZER_SUPPORTS_INIT_FOR_DLOPEN` macro which indicates if the
  feature is supported. This only true for Darwin currently.
- A `HandleDlopenInit()` function which should return true if the library
  is being loaded via `dlopen()` and
  `SANITIZER_SUPPORTS_INIT_FOR_DLOPEN` is supported. Platforms that
  support this may perform any initialization they wish inside this
  function.

Although disabling initialization is something that could potentially
apply to other sanitizers it appears to be unnecessary for other
sanitizers so this patch only makes the change for ASan.

rdar://problem/45284065

Reviewers: kubamracek, george.karpenkov, kcc, eugenis, krytarowski

Subscribers: #sanitizers, llvm-commits

Differential Revision: https://reviews.llvm.org/D54469

llvm-svn: 348078

8 files changed, 70 insertions, 0 deletions

diff --git a/compiler-rt/lib/asan/asan_fuchsia.cc b/compiler-rt/lib/asan/asan_fuchsia.cc
index 0b5bff4f565..34399c92310 100644
--- a/compiler-rt/lib/asan/asan_fuchsia.cc
+++ b/compiler-rt/lib/asan/asan_fuchsia.cc
@@ -190,6 +190,13 @@ static void ThreadExitHook(void *hook, uptr os_id) {
   AsanThread::TSDDtor(per_thread);
 }
 
+bool HandleDlopenInit() {
+  // Not supported on this platform.
+  static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN,
+                "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false");
+  return false;
+}
+
 }  // namespace __asan
 
 // These are declared (in extern "C") by <zircon/sanitizer.h>.
diff --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h
index 654878cd15f..57869497c7d 100644
--- a/compiler-rt/lib/asan/asan_internal.h
+++ b/compiler-rt/lib/asan/asan_internal.h
@@ -111,6 +111,11 @@ void *AsanDlSymNext(const char *sym);
 
 void ReserveShadowMemoryRange(uptr beg, uptr end, const char *name);
 
+// Returns `true` iff most of ASan init process should be skipped due to the
+// ASan library being loaded via `dlopen()`. Platforms may perform any
+// `dlopen()` specific initialization inside this function.
+bool HandleDlopenInit();
+
 // Add convenient macro for interface functions that may be represented as
 // weak hooks.
 #define ASAN_MALLOC_HOOK(ptr, size)                                   \
diff --git a/compiler-rt/lib/asan/asan_linux.cc b/compiler-rt/lib/asan/asan_linux.cc
index 625f32d408d..a150b1955d6 100644
--- a/compiler-rt/lib/asan/asan_linux.cc
+++ b/compiler-rt/lib/asan/asan_linux.cc
@@ -248,6 +248,13 @@ void *AsanDlSymNext(const char *sym) {
   return dlsym(RTLD_NEXT, sym);
 }
 
+bool HandleDlopenInit() {
+  // Not supported on this platform.
+  static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN,
+                "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false");
+  return false;
+}
+
 } // namespace __asan
 
 #endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD ||
diff --git a/compiler-rt/lib/asan/asan_malloc_mac.cc b/compiler-rt/lib/asan/asan_malloc_mac.cc
index 733ba2d86e1..27281f1bc83 100644
--- a/compiler-rt/lib/asan/asan_malloc_mac.cc
+++ b/compiler-rt/lib/asan/asan_malloc_mac.cc
@@ -61,4 +61,25 @@ using namespace __asan;
 
 #include "sanitizer_common/sanitizer_malloc_mac.inc"
 
+namespace COMMON_MALLOC_NAMESPACE {
+bool HandleDlopenInit() {
+  static_assert(SANITIZER_SUPPORTS_INIT_FOR_DLOPEN,
+                "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be true");
+  // We have no reliable way of knowing how we are being loaded
+  // so make it a requirement on Apple platforms to set this environment
+  // variable to indicate that we want to perform initialization via
+  // dlopen().
+  auto init_str = GetEnv("APPLE_ASAN_INIT_FOR_DLOPEN");
+  if (!init_str)
+    return false;
+  if (internal_strncmp(init_str, "1", 1) != 0)
+    return false;
+  // When we are loaded via `dlopen()` path we still initialize the malloc zone
+  // so Symbolication clients (e.g. `leaks`) that load the ASan allocator can
+  // find an initialized malloc zone.
+  InitMallocZoneFields();
+  return true;
+}
+}  // namespace COMMON_MALLOC_NAMESPACE
+
 #endif
diff --git a/compiler-rt/lib/asan/asan_rtems.cc b/compiler-rt/lib/asan/asan_rtems.cc
index a4af940057e..b48cc6a75d7 100644
--- a/compiler-rt/lib/asan/asan_rtems.cc
+++ b/compiler-rt/lib/asan/asan_rtems.cc
@@ -213,6 +213,12 @@ static void HandleExit() {
   }
 }
 
+bool HandleDlopenInit() {
+  // Not supported on this platform.
+  static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN,
+                "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false");
+  return false;
+}
 }  // namespace __asan
 
 // These are declared (in extern "C") by <some_path/sanitizer.h>.
diff --git a/compiler-rt/lib/asan/asan_rtl.cc b/compiler-rt/lib/asan/asan_rtl.cc
index 0c6390352a5..0ecbcd59487 100644
--- a/compiler-rt/lib/asan/asan_rtl.cc
+++ b/compiler-rt/lib/asan/asan_rtl.cc
@@ -396,6 +396,14 @@ static void AsanInitInternal() {
   // initialization steps look at flags().
   InitializeFlags();
 
+  // Stop performing init at this point if we are being loaded via
+  // dlopen() and the platform supports it.
+  if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) {
+    asan_init_is_running = false;
+    VReport(1, "AddressSanitizer init is being performed for dlopen().\n");
+    return;
+  }
+
   AsanCheckIncompatibleRT();
   AsanCheckDynamicRTPrereqs();
   AvoidCVE_2016_2143();
diff --git a/compiler-rt/lib/asan/asan_win.cc b/compiler-rt/lib/asan/asan_win.cc
index 5661d911cab..068f4a5d247 100644
--- a/compiler-rt/lib/asan/asan_win.cc
+++ b/compiler-rt/lib/asan/asan_win.cc
@@ -322,6 +322,13 @@ int __asan_set_seh_filter() {
   return 0;
 }
 
+bool HandleDlopenInit() {
+  // Not supported on this platform.
+  static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN,
+                "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false");
+  return false;
+}
+
 #if !ASAN_DYNAMIC
 // The CRT runs initializers in this order:
 // - C initializers, from XIA to XIZ
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
index 106a147e573..fd7228c3b33 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform.h
@@ -342,4 +342,13 @@
 #define SANITIZER_SYMBOLIZER_MARKUP 0
 #endif
 
+// Enable ability to support sanitizer initialization that is
+// compatible with the sanitizer library being loaded via
+// `dlopen()`.
+#if SANITIZER_MAC
+#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 1
+#else
+#define SANITIZER_SUPPORTS_INIT_FOR_DLOPEN 0
+#endif
+
 #endif // SANITIZER_PLATFORM_H