15 files changed, 125 insertions, 0 deletions

diff --git a/compiler-rt/lib/asan/CMakeLists.txt b/compiler-rt/lib/asan/CMakeLists.txt
index 51bda052436..0cf6888a1bd 100644
--- a/compiler-rt/lib/asan/CMakeLists.txt
+++ b/compiler-rt/lib/asan/CMakeLists.txt
@@ -13,6 +13,7 @@ set(ASAN_SOURCES
   asan_globals_win.cc
   asan_interceptors.cc
   asan_interceptors_memintrinsics.cc
+  asan_interceptors_vfork.S
   asan_linux.cc
   asan_mac.cc
   asan_malloc_linux.cc
diff --git a/compiler-rt/lib/asan/asan_interceptors.cc b/compiler-rt/lib/asan/asan_interceptors.cc
index 7f2660df4cb..e59679bec41 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cc
+++ b/compiler-rt/lib/asan/asan_interceptors.cc
@@ -579,6 +579,8 @@ INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
 }
 #endif  // ASAN_INTERCEPT___CXA_ATEXIT
 
+DEFINE_REAL(void, vfork);
+
 // ---------------------- InitializeAsanInterceptors ---------------- {{{1
 namespace __asan {
 void InitializeAsanInterceptors() {
@@ -656,6 +658,8 @@ void InitializeAsanInterceptors() {
   ASAN_INTERCEPT_FUNC(__cxa_atexit);
 #endif
 
+  __interception::GetRealFunctionAddress("vfork", (uptr *)&REAL(vfork), 0, 0);
+
   InitializePlatformInterceptors();
 
   VReport(1, "AddressSanitizer: libc interceptors initialized\n");
diff --git a/compiler-rt/lib/asan/asan_interceptors_vfork.S b/compiler-rt/lib/asan/asan_interceptors_vfork.S
new file mode 100644
index 00000000000..7f4d8c688f7
--- /dev/null
+++ b/compiler-rt/lib/asan/asan_interceptors_vfork.S
@@ -0,0 +1,7 @@
+#define COMMON_INTERCEPTOR_SPILL_AREA __asan_extra_spill_area
+#define COMMON_INTERCEPTOR_HANDLE_VFORK __asan_handle_vfork
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
+
+#if defined(__linux__)
+.section        .note.GNU-stack,"",@progbits
+#endif
diff --git a/compiler-rt/lib/asan/asan_interface.inc b/compiler-rt/lib/asan/asan_interface.inc
index 1dd9c630112..99f1db4936e 100644
--- a/compiler-rt/lib/asan/asan_interface.inc
+++ b/compiler-rt/lib/asan/asan_interface.inc
@@ -26,6 +26,7 @@ INTERFACE_FUNCTION(__asan_exp_store4)
 INTERFACE_FUNCTION(__asan_exp_store8)
 INTERFACE_FUNCTION(__asan_exp_store16)
 INTERFACE_FUNCTION(__asan_exp_storeN)
+INTERFACE_FUNCTION(__asan_extra_spill_area)
 INTERFACE_FUNCTION(__asan_get_alloc_stack)
 INTERFACE_FUNCTION(__asan_get_current_fake_stack)
 INTERFACE_FUNCTION(__asan_get_free_stack)
@@ -38,6 +39,7 @@ INTERFACE_FUNCTION(__asan_get_report_pc)
 INTERFACE_FUNCTION(__asan_get_report_sp)
 INTERFACE_FUNCTION(__asan_get_shadow_mapping)
 INTERFACE_FUNCTION(__asan_handle_no_return)
+INTERFACE_FUNCTION(__asan_handle_vfork)
 INTERFACE_FUNCTION(__asan_init)
 INTERFACE_FUNCTION(__asan_load_cxx_array_cookie)
 INTERFACE_FUNCTION(__asan_load1)
diff --git a/compiler-rt/lib/asan/asan_interface_internal.h b/compiler-rt/lib/asan/asan_interface_internal.h
index ea8750e9059..59163a3327c 100644
--- a/compiler-rt/lib/asan/asan_interface_internal.h
+++ b/compiler-rt/lib/asan/asan_interface_internal.h
@@ -249,6 +249,9 @@ extern "C" {
 
   SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
   const char* __asan_default_suppressions();
+
+  SANITIZER_INTERFACE_ATTRIBUTE void *__asan_extra_spill_area();
+  SANITIZER_INTERFACE_ATTRIBUTE void __asan_handle_vfork(void *sp);
 }  // extern "C"
 
 #endif  // ASAN_INTERFACE_INTERNAL_H
diff --git a/compiler-rt/lib/asan/asan_rtl.cc b/compiler-rt/lib/asan/asan_rtl.cc
index 67eb1a7832c..f68373809fc 100644
--- a/compiler-rt/lib/asan/asan_rtl.cc
+++ b/compiler-rt/lib/asan/asan_rtl.cc
@@ -597,6 +597,21 @@ void NOINLINE __asan_handle_no_return() {
     curr_thread->fake_stack()->HandleNoReturn();
 }
 
+void *__asan_extra_spill_area() {
+  AsanThread *t = GetCurrentThread();
+  CHECK(t);
+  return t->extra_spill_area();
+}
+
+void __asan_handle_vfork(void *sp) {
+  AsanThread *t = GetCurrentThread();
+  CHECK(t);
+  uptr PageSize = GetPageSizeCached();
+  uptr top = t->stack_top();
+  uptr bottom = ((uptr)sp - PageSize) & ~(PageSize - 1);
+  PoisonShadow(bottom, top - bottom, 0);
+}
+
 void NOINLINE __asan_set_death_callback(void (*callback)(void)) {
   SetUserDieCallback(callback);
 }
diff --git a/compiler-rt/lib/asan/asan_thread.h b/compiler-rt/lib/asan/asan_thread.h
index 1e08d577123..5a6010e855f 100644
--- a/compiler-rt/lib/asan/asan_thread.h
+++ b/compiler-rt/lib/asan/asan_thread.h
@@ -130,6 +130,8 @@ class AsanThread {
   AsanThreadLocalMallocStorage &malloc_storage() { return malloc_storage_; }
   AsanStats &stats() { return stats_; }
 
+  void *extra_spill_area() { return &extra_spill_area_; }
+
  private:
   // NOTE: There is no AsanThread constructor. It is allocated
   // via mmap() and *must* be valid in zero-initialized state.
@@ -165,6 +167,7 @@ class AsanThread {
   AsanThreadLocalMallocStorage malloc_storage_;
   AsanStats stats_;
   bool unwinding_;
+  uptr extra_spill_area_;
 };
 
 // ScopedUnwinding is a scope for stacktracing member of a context
diff --git a/compiler-rt/lib/hwasan/CMakeLists.txt b/compiler-rt/lib/hwasan/CMakeLists.txt
index 7460abc54bd..d645644124a 100644
--- a/compiler-rt/lib/hwasan/CMakeLists.txt
+++ b/compiler-rt/lib/hwasan/CMakeLists.txt
@@ -6,6 +6,7 @@ set(HWASAN_RTL_SOURCES
   hwasan_allocator.cc
   hwasan_dynamic_shadow.cc
   hwasan_interceptors.cc
+  hwasan_interceptors_vfork.S
   hwasan_linux.cc
   hwasan_memintrinsics.cc
   hwasan_poisoning.cc
diff --git a/compiler-rt/lib/hwasan/hwasan.cc b/compiler-rt/lib/hwasan/hwasan.cc
index 9c83f73bd98..3a87dea26d3 100644
--- a/compiler-rt/lib/hwasan/hwasan.cc
+++ b/compiler-rt/lib/hwasan/hwasan.cc
@@ -477,6 +477,30 @@ void __hwasan_handle_longjmp(const void *sp_dst) {
   TagMemory(sp, dst - sp, 0);
 }
 
+void __hwasan_handle_vfork(const void *sp_dst) {
+  uptr sp = (uptr)sp_dst;
+  Thread *t = GetCurrentThread();
+  CHECK(t);
+  uptr top = t->stack_top();
+  uptr bottom = t->stack_bottom();
+  static const uptr kMaxExpectedCleanupSize = 64 << 20;  // 64M
+  if (top == 0 || bottom == 0 || sp < bottom || sp >= top ||
+      sp - bottom > kMaxExpectedCleanupSize) {
+    Report(
+        "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: "
+        "stack top: %zx; current %zx; bottom: %zx \n"
+        "False positive error reports may follow\n",
+        top, sp, bottom);
+    return;
+  }
+  TagMemory(bottom, sp - bottom, 0);
+}
+
+void *__hwasan_extra_spill_area() {
+  Thread *t = GetCurrentThread();
+  return &t->vfork_spill();
+}
+
 void __hwasan_print_memory_usage() {
   InternalScopedString s(kMemoryUsageBufferSize);
   HwasanFormatMemoryUsage(s);
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors.cc b/compiler-rt/lib/hwasan/hwasan_interceptors.cc
index baecb1e0334..1884b3da368 100644
--- a/compiler-rt/lib/hwasan/hwasan_interceptors.cc
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors.cc
@@ -227,6 +227,10 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr,
 }
 #endif
 
+#if HWASAN_WITH_INTERCEPTORS
+DEFINE_REAL(void, vfork);
+#endif
+
 static void BeforeFork() {
   StackDepotLockAll();
 }
@@ -266,6 +270,7 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(fork);
 
 #if HWASAN_WITH_INTERCEPTORS
+  __interception::GetRealFunctionAddress("vfork", (uptr *)&REAL(vfork), 0, 0);
 #if !defined(__aarch64__)
   INTERCEPT_FUNCTION(pthread_create);
 #endif
diff --git a/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
new file mode 100644
index 00000000000..54b4a78db6c
--- /dev/null
+++ b/compiler-rt/lib/hwasan/hwasan_interceptors_vfork.S
@@ -0,0 +1,7 @@
+#define COMMON_INTERCEPTOR_SPILL_AREA __hwasan_extra_spill_area
+#define COMMON_INTERCEPTOR_HANDLE_VFORK __hwasan_handle_vfork
+#include "sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S"
+
+#if defined(__linux__)
+.section        .note.GNU-stack,"",@progbits
+#endif
diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h
index c2ae666533a..1b025c92342 100644
--- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h
+++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h
@@ -117,6 +117,9 @@ SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_handle_longjmp(const void *sp_dst);
 
 SANITIZER_INTERFACE_ATTRIBUTE
+void __hwasan_handle_vfork(const void *sp_dst);
+
+SANITIZER_INTERFACE_ATTRIBUTE
 u16 __sanitizer_unaligned_load16(const uu16 *p);
 
 SANITIZER_INTERFACE_ATTRIBUTE
@@ -200,6 +203,9 @@ SANITIZER_INTERFACE_ATTRIBUTE
 void *__hwasan_memset(void *s, int c, uptr n);
 SANITIZER_INTERFACE_ATTRIBUTE
 void *__hwasan_memmove(void *dest, const void *src, uptr n);
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__hwasan_extra_spill_area();
 }  // extern "C"
 
 #endif  // HWASAN_INTERFACE_INTERNAL_H
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.h b/compiler-rt/lib/hwasan/hwasan_thread.h
index 9c45adec1b1..6fa592bfac6 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.h
+++ b/compiler-rt/lib/hwasan/hwasan_thread.h
@@ -67,11 +67,14 @@ class Thread {
     Print("Thread: ");
   }
 
+  uptr &vfork_spill() { return vfork_spill_; }
+
  private:
   // NOTE: There is no Thread constructor. It is allocated
   // via mmap() and *must* be valid in zero-initialized state.
   void ClearShadowForThreadStackAndTLS();
   void Print(const char *prefix);
+  uptr vfork_spill_;
   uptr stack_top_;
   uptr stack_bottom_;
   uptr tls_begin_;
diff --git a/compiler-rt/lib/sanitizer_common/CMakeLists.txt b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
index 4d331ec5e6c..d3780457206 100644
--- a/compiler-rt/lib/sanitizer_common/CMakeLists.txt
+++ b/compiler-rt/lib/sanitizer_common/CMakeLists.txt
@@ -126,6 +126,7 @@ set(SANITIZER_IMPL_HEADERS
   sanitizer_common_interceptors.inc
   sanitizer_common_interceptors_format.inc
   sanitizer_common_interceptors_ioctl.inc
+  sanitizer_common_interceptors_vfork_aarch64.inc.S
   sanitizer_common_interface.inc
   sanitizer_common_interface_posix.inc
   sanitizer_common_syscalls.inc
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S
new file mode 100644
index 00000000000..fa20c7765c6
--- /dev/null
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_vfork_aarch64.inc.S
@@ -0,0 +1,43 @@
+#if defined(__aarch64__) && defined(__linux__)
+
+#include "sanitizer_common/sanitizer_asm.h"
+
+ASM_HIDDEN(COMMON_INTERCEPTOR_SPILL_AREA)
+
+.comm _ZN14__interception10real_vforkE,8,8
+.globl vfork
+ASM_TYPE_FUNCTION(vfork)
+vfork:
+  // Save x30 in the off-stack spill area.
+  stp     xzr, x30, [sp, #-16]!
+  bl      COMMON_INTERCEPTOR_SPILL_AREA
+  ldp     xzr, x30, [sp], 16
+  str     x30, [x0]
+
+  // Call real vfork. This may return twice. User code that runs between the first and the second return
+  // may clobber the stack frame of the interceptor; that's why it does not have a frame.
+  adrp    x0, _ZN14__interception10real_vforkE
+  ldr     x0, [x0, :lo12:_ZN14__interception10real_vforkE]
+  blr     x0
+
+  stp     x0, xzr, [sp, #-16]!
+  cmp     x0, #0
+  b.eq   .L_exit
+
+  // x0 != 0 => parent process. Clear stack shadow.
+  add    x0, sp, #16
+  bl     COMMON_INTERCEPTOR_HANDLE_VFORK
+
+.L_exit:
+  // Restore x30.
+  bl     COMMON_INTERCEPTOR_SPILL_AREA
+  ldr    x30, [x0]
+  ldp    x0, xzr, [sp], 16
+
+  ret
+ASM_SIZE(vfork)
+
+.globl __interceptor_vfork
+.set __interceptor_vfork, vfork
+
+#endif