[libFuzzer] Handle unstable edges by disregarding unstable edges

Summary:
Added a new mode within flag -handle_unstable for new unstable handling algorithm that does the following:
    When an edge is shown as unstable, copy to UnstableCounters the value 0.
    During ApplyUnstableCounters we copy back the value 0 to ModuleInline8bitCounters if the edge was unstable.

This way we would be ignoring completely features that were collected through non-determinism.
Unstable hits would be counted as if it never hit.

Reviewers: metzman, Dor1s, kcc, morehouse

Reviewed By: metzman, morehouse

Subscribers: delcypher, llvm-commits, #sanitizers

Differential Revision: https://reviews.llvm.org/D49684

llvm-svn: 337853

6 files changed, 58 insertions, 7 deletions

diff --git a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
index 4f40d6abf63..783474a39e1 100644
--- a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
@@ -619,7 +619,8 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
   Options.PrintCorpusStats = Flags.print_corpus_stats;
   Options.PrintCoverage = Flags.print_coverage;
   Options.PrintUnstableStats = Flags.print_unstable_stats;
-  if (Flags.handle_unstable)
+  if (Flags.handle_unstable == TracePC::MinUnstable ||
+      Flags.handle_unstable == TracePC::ZeroUnstable)
     Options.HandleUnstable = Flags.handle_unstable;
   Options.DumpCoverage = Flags.dump_coverage;
   if (Flags.exit_on_src_pos)
diff --git a/compiler-rt/lib/fuzzer/FuzzerFlags.def b/compiler-rt/lib/fuzzer/FuzzerFlags.def
index aa3a987b1d0..ba04bc25fd4 100644
--- a/compiler-rt/lib/fuzzer/FuzzerFlags.def
+++ b/compiler-rt/lib/fuzzer/FuzzerFlags.def
@@ -114,7 +114,9 @@ FUZZER_FLAG_INT(handle_unstable, 0, "Experimental."
                    " Executes every input 3 times in total if a unique feature"
                    " is found during the first execution."
                    " If 1, we only use the minimum hit count from the 3 runs"
-                   " to determine whether an input is interesting.")
+                   " to determine whether an input is interesting."
+                   " If 2, we disregard edges that are found unstable for"
+                   " feature collection.")
 FUZZER_FLAG_INT(print_unstable_stats, 0, "Experimental."
 				  " If 1, print unstable statistics at exit.")
 FUZZER_FLAG_INT(handle_segv, 1, "If 1, try to intercept SIGSEGV.")
diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
index e63ee7361c5..4bc88365a0b 100644
--- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
@@ -472,7 +472,8 @@ void Fuzzer::CheckForUnstableCounters(const uint8_t *Data, size_t Size) {
   TPC.UpdateUnstableCounters(Options.HandleUnstable);
 
   // Move minimum hit counts back to ModuleInline8bitCounters
-  if (Options.HandleUnstable)
+  if (Options.HandleUnstable == TracePC::MinUnstable ||
+      Options.HandleUnstable == TracePC::ZeroUnstable)
     TPC.ApplyUnstableCounters();
 }
 
diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
index 859ff633373..29ffc8e34fc 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
@@ -83,11 +83,14 @@ void TracePC::InitializeUnstableCounters() {
 // and records differences as unstable edges.
 void TracePC::UpdateUnstableCounters(int UnstableMode) {
   IterateInline8bitCounters([&](int i, int j, int UnstableIdx) {
-    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter)
+    if (ModuleCounters[i].Start[j] != UnstableCounters[UnstableIdx].Counter) {
       UnstableCounters[UnstableIdx].IsUnstable = true;
-    if (UnstableMode &&
-        ModuleCounters[i].Start[j] < UnstableCounters[UnstableIdx].Counter)
-      UnstableCounters[UnstableIdx].Counter = ModuleCounters[i].Start[j];
+      if (UnstableMode == ZeroUnstable)
+        UnstableCounters[UnstableIdx].Counter = 0;
+      else if (UnstableMode == MinUnstable)
+        UnstableCounters[UnstableIdx].Counter = std::min(
+            ModuleCounters[i].Start[j], UnstableCounters[UnstableIdx].Counter);
+    }
   });
 }
 
diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.h b/compiler-rt/lib/fuzzer/FuzzerTracePC.h
index a051568c41d..097ba69bdc0 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.h
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.h
@@ -74,6 +74,11 @@ class TracePC {
   // How many bits of PC are used from __sanitizer_cov_trace_pc.
   static const size_t kTracePcBits = 18;
 
+  enum HandleUnstableOptions {
+    MinUnstable = 1,
+    ZeroUnstable = 2,
+  };
+
   void HandleInit(uint32_t *Start, uint32_t *Stop);
   void HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop);
   void HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop);
diff --git a/compiler-rt/test/fuzzer/handle-unstable.test b/compiler-rt/test/fuzzer/handle-unstable.test
new file mode 100644
index 00000000000..2116d9a5e4d
--- /dev/null
+++ b/compiler-rt/test/fuzzer/handle-unstable.test
@@ -0,0 +1,39 @@
+RUN: %cpp_compiler %S/PrintUnstableStatsTest.cpp -o %t-HandleUnstableTest
+
+; Normal
+RUN: %run %t-HandleUnstableTest -print_coverage=1 -runs=100000 2>&1 | FileCheck %s --check-prefix=NORMAL
+NORMAL-DAG: det0()
+NORMAL-DAG: det1()
+NORMAL-DAG: det2()
+NORMAL-DAG: det3()
+NORMAL-DAG: det4()
+NORMAL-DAG: ini0()
+NORMAL-DAG: ini1()
+NORMAL-DAG: ini2()
+NORMAL-DAG: t0()
+NORMAL-DAG: t1()
+NORMAL-DAG: t2()
+NORMAL-DAG: t3()
+NORMAL-DAG: t4()
+
+; MinUnstable
+RUN: %run %t-HandleUnstableTest -print_coverage=1 -handle_unstable=1 -runs=100000 2>&1 | FileCheck %s --check-prefix=MIN
+MIN-NOT: ini0()
+MIN-NOT: ini1()
+MIN-NOT: ini2()
+MIN: det0()
+MIN: det1()
+MIN: det2()
+MIN: det3()
+MIN: det4()
+
+; ZeroUnstable
+RUN: %run %t-HandleUnstableTest -print_coverage=1 -handle_unstable=2 -runs=1 2>&1 | FileCheck %s --check-prefix=ZERO
+ZERO-NOT: ini0()
+ZERO-NOT: ini1()
+ZERO-NOT: ini2()
+ZERO: det0()
+ZERO: det1()
+ZERO: det2()
+ZERO: det3()
+ZERO: det4()