[libFuzzer] enhance -rss_limit_mb and enable by default. Now it will print the OOM reproducer.

llvm-svn: 268821

9 files changed, 88 insertions, 15 deletions

diff --git a/llvm/docs/LibFuzzer.rst b/llvm/docs/LibFuzzer.rst
index d2d02e89939..fb6a10dc6ea 100644
--- a/llvm/docs/LibFuzzer.rst
+++ b/llvm/docs/LibFuzzer.rst
@@ -201,6 +201,12 @@ The most important command line options are:
 ``-timeout``
   Timeout in seconds, default 1200. If an input takes longer than this timeout,
   the process is treated as a failure case.
+``-rss_limit_mb``
+  Memory usage limit in Mb, default 2048. Use 0 to disable the limit.
+  If an input requires more than this amount of RSS memory to execute,
+  the process is treated as a failure case.
+  The limit is checked in a separate thread every second.
+  If running w/o ASAN/MSAN, you may use 'ulimit -v' instead.
 ``-timeout_exitcode``
   Exit code (default 77) to emit when terminating due to timeout, when
   ``-abort_on_timeout`` is not set.
diff --git a/llvm/lib/Fuzzer/FuzzerDriver.cpp b/llvm/lib/Fuzzer/FuzzerDriver.cpp
index 38e19689957..9bd991a0828 100644
--- a/llvm/lib/Fuzzer/FuzzerDriver.cpp
+++ b/llvm/lib/Fuzzer/FuzzerDriver.cpp
@@ -189,7 +189,7 @@ static std::mutex Mu;
 
 static void PulseThread() {
   while (true) {
-    std::this_thread::sleep_for(std::chrono::seconds(600));
+    SleepSeconds(600);
     std::lock_guard<std::mutex> Lock(Mu);
     Printf("pulse...\n");
   }
@@ -236,10 +236,10 @@ static int RunInMultipleProcesses(const std::vector<std::string> &Args,
 
 static void RssThread(Fuzzer *F, size_t RssLimitMb) {
   while (true) {
-    std::this_thread::sleep_for(std::chrono::seconds(1));
+    SleepSeconds(1);
     size_t Peak = GetPeakRSSMb();
     if (Peak > RssLimitMb)
-      F->RssLimitCallback(Peak, RssLimitMb);
+      F->RssLimitCallback();
   }
 }
 
@@ -310,6 +310,7 @@ static int FuzzerDriver(const std::vector<std::string> &Args,
   Options.OnlyASCII = Flags.only_ascii;
   Options.OutputCSV = Flags.output_csv;
   Options.DetectLeaks = Flags.detect_leaks;
+  Options.RssLimitMb = Flags.rss_limit_mb;
   if (Flags.runs >= 0)
     Options.MaxNumberOfRuns = Flags.runs;
   if (!Inputs->empty())
diff --git a/llvm/lib/Fuzzer/FuzzerFlags.def b/llvm/lib/Fuzzer/FuzzerFlags.def
index 86597cc6808..1024fa6f82e 100644
--- a/llvm/lib/Fuzzer/FuzzerFlags.def
+++ b/llvm/lib/Fuzzer/FuzzerFlags.def
@@ -81,7 +81,7 @@ FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; "
     "Be careful, this will also close e.g. asan's stderr/stdout.")
 FUZZER_FLAG_INT(detect_leaks, 1, "If 1, and if LeakSanitizer is enabled "
     "try to detect memory leaks during fuzzing (i.e. not only at shut down).")
-FUZZER_FLAG_INT(rss_limit_mb, 0, "If non-zero, the fuzzer will exit upon"
+FUZZER_FLAG_INT(rss_limit_mb, 2048, "If non-zero, the fuzzer will exit upon"
     "reaching this limit of RSS memory usage.")
 
 FUZZER_DEPRECATED_FLAG(exit_on_first)
diff --git a/llvm/lib/Fuzzer/FuzzerInternal.h b/llvm/lib/Fuzzer/FuzzerInternal.h
index 75f058b7d1c..9b99767fdf2 100644
--- a/llvm/lib/Fuzzer/FuzzerInternal.h
+++ b/llvm/lib/Fuzzer/FuzzerInternal.h
@@ -109,6 +109,8 @@ bool IsASCII(const Unit &U);
 
 int NumberOfCpuCores();
 int GetPid();
+int SignalToMainThread();
+void SleepSeconds(int Seconds);
 
 // Clears the current PC Map.
 void PcMapResetCurrent();
@@ -283,6 +285,7 @@ public:
     int TimeoutExitCode = 77;
     int ErrorExitCode = 77;
     int MaxTotalTimeSec = 0;
+    int RssLimitMb = 0;
     bool DoCrossOver = true;
     int MutateDepth = 5;
     bool UseCounters = false;
@@ -353,7 +356,7 @@ public:
   MutationDispatcher &GetMD() { return MD; }
   void PrintFinalStats();
   void SetMaxLen(size_t MaxLen);
-  void RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb);
+  void RssLimitCallback();
 
 private:
   void AlarmCallback();
@@ -397,6 +400,7 @@ private:
 
   uint8_t *CurrentUnitData = nullptr;
   size_t CurrentUnitSize = 0;
+  bool InOOMState = false;
 
   size_t TotalNumberOfRuns = 0;
   size_t NumberOfNewUnitsAdded = 0;
diff --git a/llvm/lib/Fuzzer/FuzzerLoop.cpp b/llvm/lib/Fuzzer/FuzzerLoop.cpp
index e02ebcf6b4a..6a7fe7d2bf4 100644
--- a/llvm/lib/Fuzzer/FuzzerLoop.cpp
+++ b/llvm/lib/Fuzzer/FuzzerLoop.cpp
@@ -153,6 +153,20 @@ void Fuzzer::InterruptCallback() {
 NO_SANITIZE_MEMORY
 void Fuzzer::AlarmCallback() {
   assert(Options.UnitTimeoutSec > 0);
+  if (InOOMState) {
+    Printf("==%d== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
+           GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
+    Printf("   To change the out-of-memory limit use -rss_limit_mb=<N>\n");
+    if (CurrentUnitSize && CurrentUnitData) {
+      DumpCurrentUnit("oom-");
+      if (__sanitizer_print_stack_trace)
+        __sanitizer_print_stack_trace();
+    }
+    Printf("SUMMARY: libFuzzer: out-of-memory\n");
+    PrintFinalStats();
+    _Exit(Options.ErrorExitCode); // Stop right now.
+  }
+
   if (!CurrentUnitSize)
     return; // We have not started running units yet.
   size_t Seconds =
@@ -176,15 +190,13 @@ void Fuzzer::AlarmCallback() {
   }
 }
 
-void Fuzzer::RssLimitCallback(size_t RssPeakMb, size_t RssLimitMb) {
-  Printf("==%d== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
-         GetPid(), RssPeakMb, RssLimitMb);
-  Printf("*****************************************************************\n");
-  Printf("** Experimental! TODO: dump the stack trace and the reproducer **\n");
-  Printf("*****************************************************************\n");
-  Printf("SUMMARY: libFuzzer: out-of-memory\n");
-  PrintFinalStats();
-  _Exit(Options.ErrorExitCode); // Stop right now.
+void Fuzzer::RssLimitCallback() {
+  InOOMState = true;
+  SignalToMainThread();
+  SleepSeconds(5);
+  Printf("Signal to main thread failed (non-linux?). Exiting.\n");
+  _Exit(Options.ErrorExitCode);
+  return;
 }
 
 void Fuzzer::PrintStats(const char *Where, const char *End) {
diff --git a/llvm/lib/Fuzzer/FuzzerUtil.cpp b/llvm/lib/Fuzzer/FuzzerUtil.cpp
index d5335616a83..88b18d704c2 100644
--- a/llvm/lib/Fuzzer/FuzzerUtil.cpp
+++ b/llvm/lib/Fuzzer/FuzzerUtil.cpp
@@ -14,12 +14,16 @@
 #include <iomanip>
 #include <sys/resource.h>
 #include <sys/time.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
 #include <cassert>
+#include <chrono>
 #include <cstring>
 #include <signal.h>
 #include <sstream>
 #include <unistd.h>
 #include <errno.h>
+#include <thread>
 
 namespace fuzzer {
 
@@ -214,8 +218,18 @@ bool ParseDictionaryFile(const std::string &Text, std::vector<Unit> *Units) {
   return true;
 }
 
-int GetPid() { return getpid(); }
+void SleepSeconds(int Seconds) {
+  std::this_thread::sleep_for(std::chrono::seconds(Seconds));
+}
 
+int GetPid() { return getpid(); }
+int SignalToMainThread() {
+#ifdef __linux__
+  return syscall(SYS_tgkill, GetPid(), GetPid(), SIGALRM);
+#else
+  return 0;
+#endif
+}
 
 std::string Base64(const Unit &U) {
   static const char Table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
diff --git a/llvm/lib/Fuzzer/test/CMakeLists.txt b/llvm/lib/Fuzzer/test/CMakeLists.txt
index ff1a7349f1a..81a996930f4 100644
--- a/llvm/lib/Fuzzer/test/CMakeLists.txt
+++ b/llvm/lib/Fuzzer/test/CMakeLists.txt
@@ -26,6 +26,7 @@ set(Tests
   LeakTimeoutTest
   NullDerefTest
   NthRunCrashTest
+  OutOfMemoryTest
   RepeatedMemcmp
   SimpleCmpTest
   SimpleDictionaryTest
diff --git a/llvm/lib/Fuzzer/test/OutOfMemoryTest.cpp b/llvm/lib/Fuzzer/test/OutOfMemoryTest.cpp
new file mode 100644
index 00000000000..e5c9f0a038f
--- /dev/null
+++ b/llvm/lib/Fuzzer/test/OutOfMemoryTest.cpp
@@ -0,0 +1,31 @@
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+
+// Tests OOM handling.
+#include <assert.h>
+#include <cstdint>
+#include <cstdlib>
+#include <cstddef>
+#include <cstring>
+#include <iostream>
+#include <unistd.h>
+
+static volatile char *SinkPtr;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  if (Size > 0 && Data[0] == 'H') {
+    if (Size > 1 && Data[1] == 'i') {
+      if (Size > 2 && Data[2] == '!') {
+        while (true) {
+          size_t kSize = 1 << 28;
+          char *p = new char[kSize];
+          memset(p, 0, kSize);
+          SinkPtr = p;
+          sleep(1);
+        }
+      }
+    }
+  }
+  return 0;
+}
+
diff --git a/llvm/lib/Fuzzer/test/fuzzer-oom.test b/llvm/lib/Fuzzer/test/fuzzer-oom.test
new file mode 100644
index 00000000000..4cdff2142fd
--- /dev/null
+++ b/llvm/lib/Fuzzer/test/fuzzer-oom.test
@@ -0,0 +1,4 @@
+RUN: not LLVMFuzzer-OutOfMemoryTest -rss_limit_mb=10 2>&1 | FileCheck %s
+CHECK: ERROR: libFuzzer: out-of-memory (used: {{.*}}; limit: 10Mb)
+CHECK: Test unit written to ./oom-
+SUMMARY: libFuzzer: out-of-memory