summaryrefslogtreecommitdiffstats
path: root/llvm/utils/benchmark/test/output_test_helper.cc
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/utils/benchmark/test/output_test_helper.cc')
-rw-r--r--llvm/utils/benchmark/test/output_test_helper.cc423
1 files changed, 423 insertions, 0 deletions
diff --git a/llvm/utils/benchmark/test/output_test_helper.cc b/llvm/utils/benchmark/test/output_test_helper.cc
new file mode 100644
index 00000000000..6b18fe43593
--- /dev/null
+++ b/llvm/utils/benchmark/test/output_test_helper.cc
@@ -0,0 +1,423 @@
+#include <iostream>
+#include <map>
+#include <memory>
+#include <sstream>
+#include <cstring>
+
+#include "../src/check.h" // NOTE: check.h is for internal use only!
+#include "../src/re.h" // NOTE: re.h is for internal use only
+#include "output_test.h"
+#include "../src/benchmark_api_internal.h"
+
+// ========================================================================= //
+// ------------------------------ Internals -------------------------------- //
+// ========================================================================= //
+namespace internal {
+namespace {
+
+using TestCaseList = std::vector<TestCase>;
+
+// Use a vector because the order elements are added matters during iteration.
+// std::map/unordered_map don't guarantee that.
+// For example:
+// SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
+// Substitute("%HelloWorld") // Always expands to Hello.
+using SubMap = std::vector<std::pair<std::string, std::string>>;
+
+TestCaseList& GetTestCaseList(TestCaseID ID) {
+ // Uses function-local statics to ensure initialization occurs
+ // before first use.
+ static TestCaseList lists[TC_NumID];
+ return lists[ID];
+}
+
+SubMap& GetSubstitutions() {
+ // Don't use 'dec_re' from header because it may not yet be initialized.
+ static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
+ static SubMap map = {
+ {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
+ // human-readable float
+ {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"},
+ {"%int", "[ ]*[0-9]+"},
+ {" %s ", "[ ]+"},
+ {"%time", "[ ]*[0-9]{1,6} ns"},
+ {"%console_report", "[ ]*[0-9]{1,6} ns [ ]*[0-9]{1,6} ns [ ]*[0-9]+"},
+ {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
+ {"%csv_header",
+ "name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
+ "items_per_second,label,error_occurred,error_message"},
+ {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
+ {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
+ {"%csv_bytes_report",
+ "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
+ {"%csv_items_report",
+ "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
+ {"%csv_bytes_items_report",
+ "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re +
+ "," + safe_dec_re + ",,,"},
+ {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
+ {"%csv_label_report_end", ",,"}};
+ return map;
+}
+
+std::string PerformSubstitutions(std::string source) {
+ SubMap const& subs = GetSubstitutions();
+ using SizeT = std::string::size_type;
+ for (auto const& KV : subs) {
+ SizeT pos;
+ SizeT next_start = 0;
+ while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
+ next_start = pos + KV.second.size();
+ source.replace(pos, KV.first.size(), KV.second);
+ }
+ }
+ return source;
+}
+
+void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
+ TestCaseList const& not_checks) {
+ std::string first_line;
+ bool on_first = true;
+ std::string line;
+ while (remaining_output.eof() == false) {
+ CHECK(remaining_output.good());
+ std::getline(remaining_output, line);
+ if (on_first) {
+ first_line = line;
+ on_first = false;
+ }
+ for (const auto& NC : not_checks) {
+ CHECK(!NC.regex->Match(line))
+ << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
+ << NC.regex_str << "\""
+ << "\n actual regex string \"" << TC.substituted_regex << "\""
+ << "\n started matching near: " << first_line;
+ }
+ if (TC.regex->Match(line)) return;
+ CHECK(TC.match_rule != MR_Next)
+ << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
+ << "\""
+ << "\n actual regex string \"" << TC.substituted_regex << "\""
+ << "\n started matching near: " << first_line;
+ }
+ CHECK(remaining_output.eof() == false)
+ << "End of output reached before match for regex \"" << TC.regex_str
+ << "\" was found"
+ << "\n actual regex string \"" << TC.substituted_regex << "\""
+ << "\n started matching near: " << first_line;
+}
+
+void CheckCases(TestCaseList const& checks, std::stringstream& output) {
+ std::vector<TestCase> not_checks;
+ for (size_t i = 0; i < checks.size(); ++i) {
+ const auto& TC = checks[i];
+ if (TC.match_rule == MR_Not) {
+ not_checks.push_back(TC);
+ continue;
+ }
+ CheckCase(output, TC, not_checks);
+ not_checks.clear();
+ }
+}
+
+class TestReporter : public benchmark::BenchmarkReporter {
+ public:
+ TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
+ : reporters_(reps) {}
+
+ virtual bool ReportContext(const Context& context) {
+ bool last_ret = false;
+ bool first = true;
+ for (auto rep : reporters_) {
+ bool new_ret = rep->ReportContext(context);
+ CHECK(first || new_ret == last_ret)
+ << "Reports return different values for ReportContext";
+ first = false;
+ last_ret = new_ret;
+ }
+ (void)first;
+ return last_ret;
+ }
+
+ void ReportRuns(const std::vector<Run>& report) {
+ for (auto rep : reporters_) rep->ReportRuns(report);
+ }
+ void Finalize() {
+ for (auto rep : reporters_) rep->Finalize();
+ }
+
+ private:
+ std::vector<benchmark::BenchmarkReporter *> reporters_;
+};
+}
+
+} // end namespace internal
+
+// ========================================================================= //
+// -------------------------- Results checking ----------------------------- //
+// ========================================================================= //
+
+namespace internal {
+
+// Utility class to manage subscribers for checking benchmark results.
+// It works by parsing the CSV output to read the results.
+class ResultsChecker {
+ public:
+
+ struct PatternAndFn : public TestCase { // reusing TestCase for its regexes
+ PatternAndFn(const std::string& rx, ResultsCheckFn fn_)
+ : TestCase(rx), fn(fn_) {}
+ ResultsCheckFn fn;
+ };
+
+ std::vector< PatternAndFn > check_patterns;
+ std::vector< Results > results;
+ std::vector< std::string > field_names;
+
+ void Add(const std::string& entry_pattern, ResultsCheckFn fn);
+
+ void CheckResults(std::stringstream& output);
+
+ private:
+
+ void SetHeader_(const std::string& csv_header);
+ void SetValues_(const std::string& entry_csv_line);
+
+ std::vector< std::string > SplitCsv_(const std::string& line);
+
+};
+
+// store the static ResultsChecker in a function to prevent initialization
+// order problems
+ResultsChecker& GetResultsChecker() {
+ static ResultsChecker rc;
+ return rc;
+}
+
+// add a results checker for a benchmark
+void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) {
+ check_patterns.emplace_back(entry_pattern, fn);
+}
+
+// check the results of all subscribed benchmarks
+void ResultsChecker::CheckResults(std::stringstream& output) {
+ // first reset the stream to the start
+ {
+ auto start = std::ios::streampos(0);
+ // clear before calling tellg()
+ output.clear();
+ // seek to zero only when needed
+ if(output.tellg() > start) output.seekg(start);
+ // and just in case
+ output.clear();
+ }
+ // now go over every line and publish it to the ResultsChecker
+ std::string line;
+ bool on_first = true;
+ while (output.eof() == false) {
+ CHECK(output.good());
+ std::getline(output, line);
+ if (on_first) {
+ SetHeader_(line); // this is important
+ on_first = false;
+ continue;
+ }
+ SetValues_(line);
+ }
+ // finally we can call the subscribed check functions
+ for(const auto& p : check_patterns) {
+ VLOG(2) << "--------------------------------\n";
+ VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n";
+ for(const auto& r : results) {
+ if(!p.regex->Match(r.name)) {
+ VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n";
+ continue;
+ } else {
+ VLOG(2) << p.regex_str << " is matched by " << r.name << "\n";
+ }
+ VLOG(1) << "Checking results of " << r.name << ": ... \n";
+ p.fn(r);
+ VLOG(1) << "Checking results of " << r.name << ": OK.\n";
+ }
+ }
+}
+
+// prepare for the names in this header
+void ResultsChecker::SetHeader_(const std::string& csv_header) {
+ field_names = SplitCsv_(csv_header);
+}
+
+// set the values for a benchmark
+void ResultsChecker::SetValues_(const std::string& entry_csv_line) {
+ if(entry_csv_line.empty()) return; // some lines are empty
+ CHECK(!field_names.empty());
+ auto vals = SplitCsv_(entry_csv_line);
+ CHECK_EQ(vals.size(), field_names.size());
+ results.emplace_back(vals[0]); // vals[0] is the benchmark name
+ auto &entry = results.back();
+ for (size_t i = 1, e = vals.size(); i < e; ++i) {
+ entry.values[field_names[i]] = vals[i];
+ }
+}
+
+// a quick'n'dirty csv splitter (eliminating quotes)
+std::vector< std::string > ResultsChecker::SplitCsv_(const std::string& line) {
+ std::vector< std::string > out;
+ if(line.empty()) return out;
+ if(!field_names.empty()) out.reserve(field_names.size());
+ size_t prev = 0, pos = line.find_first_of(','), curr = pos;
+ while(pos != line.npos) {
+ CHECK(curr > 0);
+ if(line[prev] == '"') ++prev;
+ if(line[curr-1] == '"') --curr;
+ out.push_back(line.substr(prev, curr-prev));
+ prev = pos + 1;
+ pos = line.find_first_of(',', pos + 1);
+ curr = pos;
+ }
+ curr = line.size();
+ if(line[prev] == '"') ++prev;
+ if(line[curr-1] == '"') --curr;
+ out.push_back(line.substr(prev, curr-prev));
+ return out;
+}
+
+} // end namespace internal
+
+size_t AddChecker(const char* bm_name, ResultsCheckFn fn)
+{
+ auto &rc = internal::GetResultsChecker();
+ rc.Add(bm_name, fn);
+ return rc.results.size();
+}
+
+int Results::NumThreads() const {
+ auto pos = name.find("/threads:");
+ if(pos == name.npos) return 1;
+ auto end = name.find('/', pos + 9);
+ std::stringstream ss;
+ ss << name.substr(pos + 9, end);
+ int num = 1;
+ ss >> num;
+ CHECK(!ss.fail());
+ return num;
+}
+
+double Results::GetTime(BenchmarkTime which) const {
+ CHECK(which == kCpuTime || which == kRealTime);
+ const char *which_str = which == kCpuTime ? "cpu_time" : "real_time";
+ double val = GetAs< double >(which_str);
+ auto unit = Get("time_unit");
+ CHECK(unit);
+ if(*unit == "ns") {
+ return val * 1.e-9;
+ } else if(*unit == "us") {
+ return val * 1.e-6;
+ } else if(*unit == "ms") {
+ return val * 1.e-3;
+ } else if(*unit == "s") {
+ return val;
+ } else {
+ CHECK(1 == 0) << "unknown time unit: " << *unit;
+ return 0;
+ }
+}
+
+// ========================================================================= //
+// -------------------------- Public API Definitions------------------------ //
+// ========================================================================= //
+
+TestCase::TestCase(std::string re, int rule)
+ : regex_str(std::move(re)),
+ match_rule(rule),
+ substituted_regex(internal::PerformSubstitutions(regex_str)),
+ regex(std::make_shared<benchmark::Regex>()) {
+ std::string err_str;
+ regex->Init(substituted_regex,& err_str);
+ CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
+ << "\""
+ << "\n originally \"" << regex_str << "\""
+ << "\n got error: " << err_str;
+}
+
+int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
+ auto& L = internal::GetTestCaseList(ID);
+ L.insert(L.end(), il);
+ return 0;
+}
+
+int SetSubstitutions(
+ std::initializer_list<std::pair<std::string, std::string>> il) {
+ auto& subs = internal::GetSubstitutions();
+ for (auto KV : il) {
+ bool exists = false;
+ KV.second = internal::PerformSubstitutions(KV.second);
+ for (auto& EKV : subs) {
+ if (EKV.first == KV.first) {
+ EKV.second = std::move(KV.second);
+ exists = true;
+ break;
+ }
+ }
+ if (!exists) subs.push_back(std::move(KV));
+ }
+ return 0;
+}
+
+void RunOutputTests(int argc, char* argv[]) {
+ using internal::GetTestCaseList;
+ benchmark::Initialize(&argc, argv);
+ auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/true);
+ benchmark::ConsoleReporter CR(options);
+ benchmark::JSONReporter JR;
+ benchmark::CSVReporter CSVR;
+ struct ReporterTest {
+ const char* name;
+ std::vector<TestCase>& output_cases;
+ std::vector<TestCase>& error_cases;
+ benchmark::BenchmarkReporter& reporter;
+ std::stringstream out_stream;
+ std::stringstream err_stream;
+
+ ReporterTest(const char* n, std::vector<TestCase>& out_tc,
+ std::vector<TestCase>& err_tc,
+ benchmark::BenchmarkReporter& br)
+ : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
+ reporter.SetOutputStream(&out_stream);
+ reporter.SetErrorStream(&err_stream);
+ }
+ } TestCases[] = {
+ {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
+ GetTestCaseList(TC_ConsoleErr), CR},
+ {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
+ JR},
+ {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
+ CSVR},
+ };
+
+ // Create the test reporter and run the benchmarks.
+ std::cout << "Running benchmarks...\n";
+ internal::TestReporter test_rep({&CR, &JR, &CSVR});
+ benchmark::RunSpecifiedBenchmarks(&test_rep);
+
+ for (auto& rep_test : TestCases) {
+ std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
+ std::string banner(msg.size() - 1, '-');
+ std::cout << banner << msg << banner << "\n";
+
+ std::cerr << rep_test.err_stream.str();
+ std::cout << rep_test.out_stream.str();
+
+ internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
+ internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
+
+ std::cout << "\n";
+ }
+
+ // now that we know the output is as expected, we can dispatch
+ // the checks to subscribees.
+ auto &csv = TestCases[2];
+ // would use == but gcc spits a warning
+ CHECK(std::strcmp(csv.name, "CSVReporter") == 0);
+ internal::GetResultsChecker().CheckResults(csv.out_stream);
+}
OpenPOWER on IntegriCloud