[libFuzzer] Optimize handle unstable checks by reducing iterations

Summary:
We only run the 3rd check if 2nd check finds unstable edges. 
3rd UpdateUnstableCounters is now merged with ApplyUnstableCounters to only run 1 iteration.

Patch by Kyungtak Woo (@kevinwkt).

Reviewers: Dor1s, metzman, morehouse

Reviewed By: Dor1s, morehouse

Subscribers: delcypher, #sanitizers, llvm-commits, kcc

Differential Revision: https://reviews.llvm.org/D50411

llvm-svn: 339249

3 files changed, 21 insertions, 16 deletions

diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
index 23fcb8a402d..c7b13d1e507 100644
--- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
@@ -466,16 +466,11 @@ void Fuzzer::CheckForUnstableCounters(const uint8_t *Data, size_t Size) {
 
   // First Rerun
   CBSetupAndRun();
-  TPC.UpdateUnstableCounters(Options.HandleUnstable);
-
-  // Second Rerun
-  CBSetupAndRun();
-  TPC.UpdateUnstableCounters(Options.HandleUnstable);
-
-  // Move minimum hit counts back to ModuleInline8bitCounters
-  if (Options.HandleUnstable == TracePC::MinUnstable ||
-      Options.HandleUnstable == TracePC::ZeroUnstable)
-    TPC.ApplyUnstableCounters();
+  if (TPC.UpdateUnstableCounters(Options.HandleUnstable)) {
+    // Second Rerun
+    CBSetupAndRun();
+    TPC.UpdateAndApplyUnstableCounters(Options.HandleUnstable);
+  }
 }
 
 bool Fuzzer::RunOne(const uint8_t *Data, size_t Size, bool MayDeleteFile,
diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
index 7874f1315aa..1aba816e832 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
@@ -81,9 +81,11 @@ void TracePC::InitializeUnstableCounters() {
 
 // Compares the current counters with counters from previous runs
 // and records differences as unstable edges.
-void TracePC::UpdateUnstableCounters(int UnstableMode) {
+bool TracePC::UpdateUnstableCounters(int UnstableMode) {
+  bool Updated = false;
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
     if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) {
+      Updated = true;
       UnstableCounters[UnstableIdx].IsUnstable = true;
       if (UnstableMode == ZeroUnstable)
         UnstableCounters[UnstableIdx].Counter = 0;
@@ -92,12 +94,20 @@ void TracePC::UpdateUnstableCounters(int UnstableMode) {
             ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter);
     }
   });
+  return Updated;
 }
 
-// Moves the minimum hit counts to ModuleCounters.
-void TracePC::ApplyUnstableCounters() {
+// Updates and applies unstable counters to ModuleCounters in single iteration
+void TracePC::UpdateAndApplyUnstableCounters(int UnstableMode) {
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
-    ModuleCounters[i].Start[j] = UnstableCounters[UnstableIdx].Counter;
+    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) {
+      UnstableCounters[UnstableIdx].IsUnstable = true;
+      if (UnstableMode == ZeroUnstable)
+        ModuleCounters[i].Start[j] = 0;
+      else if (UnstableMode == MinUnstable)
+        ModuleCounters[i].Start[j] = std::min(
+            ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter);
+    }
   });
 }
 
diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.h b/compiler-rt/lib/fuzzer/FuzzerTracePC.h
index 097ba69bdc0..39f97acdb07 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.h
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.h
@@ -143,8 +143,8 @@ class TracePC {
   bool ObservedFocusFunction();
 
   void InitializeUnstableCounters();
-  void UpdateUnstableCounters(int UnstableMode);
-  void ApplyUnstableCounters();
+  bool UpdateUnstableCounters(int UnstableMode);
+  void UpdateAndApplyUnstableCounters(int UnstableMode);
 
 private:
   struct UnstableEdge {