[libFuzzer] Optimize handle unstable checks by reducing iterations

Summary:
We only run the 3rd check if 2nd check finds unstable edges. 
3rd UpdateUnstableCounters is now merged with ApplyUnstableCounters to only run 1 iteration.

Patch by Kyungtak Woo (@kevinwkt).

Reviewers: Dor1s, metzman, morehouse

Reviewed By: Dor1s, morehouse

Subscribers: delcypher, #sanitizers, llvm-commits, kcc

Differential Revision: https://reviews.llvm.org/D50411

llvm-svn: 339249

1 files changed, 14 insertions, 4 deletions

diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
index 7874f1315aa..1aba816e832 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
@@ -81,9 +81,11 @@ void TracePC::InitializeUnstableCounters() {
 
 // Compares the current counters with counters from previous runs
 // and records differences as unstable edges.
-void TracePC::UpdateUnstableCounters(int UnstableMode) {
+bool TracePC::UpdateUnstableCounters(int UnstableMode) {
+  bool Updated = false;
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
     if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) {
+      Updated = true;
       UnstableCounters[UnstableIdx].IsUnstable = true;
       if (UnstableMode == ZeroUnstable)
         UnstableCounters[UnstableIdx].Counter = 0;
@@ -92,12 +94,20 @@ void TracePC::UpdateUnstableCounters(int UnstableMode) {
             ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter);
     }
   });
+  return Updated;
 }
 
-// Moves the minimum hit counts to ModuleCounters.
-void TracePC::ApplyUnstableCounters() {
+// Updates and applies unstable counters to ModuleCounters in single iteration
+void TracePC::UpdateAndApplyUnstableCounters(int UnstableMode) {
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
-    ModuleCounters[i].Start[j] = UnstableCounters[UnstableIdx].Counter;
+    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) {
+      UnstableCounters[UnstableIdx].IsUnstable = true;
+      if (UnstableMode == ZeroUnstable)
+        ModuleCounters[i].Start[j] = 0;
+      else if (UnstableMode == MinUnstable)
+        ModuleCounters[i].Start[j] = std::min(
+            ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter);
+    }
   });
 }