""" The LLVM Compiler Infrastructure This file is distributed under the University of Illinois Open Source License. See LICENSE.TXT for details. Provides basic test result output. This is intended to be suitable for normal LLDB test run output when no other option is specified. """ from __future__ import print_function from . import test_results class BasicResultsFormatter(test_results.ResultsFormatter): """Provides basic test result output.""" @classmethod def arg_parser(cls): """@return arg parser used to parse formatter-specific options.""" parser = super(BasicResultsFormatter, cls).arg_parser() parser.add_argument( "--assert-on-unknown-events", action="store_true", help=('cause unknown test events to generate ' 'a python assert. Default is to ignore.')) return parser def __init__(self, out_file, options): """Initializes the BasicResultsFormatter instance. @param out_file file-like object where formatted output is written. @param options_dict specifies a dictionary of options for the formatter. """ # Initialize the parent super(BasicResultsFormatter, self).__init__(out_file, options) # self.result_event will store the most current result_event # by test method self.result_events = {} self.test_method_rerun_count = 0 def handle_event(self, test_event): super(BasicResultsFormatter, self).handle_event(test_event) if test_event is None: return event_type = test_event["event"] if event_type is None: return if event_type == "terminate": self._finish_output() elif event_type == "test_start": self.track_start_time( test_event["test_class"], test_event["test_name"], test_event["event_time"]) elif event_type == "test_result": # Build the test key. test_key = test_event.get("test_filename", None) if test_key is None: raise Exception( "failed to find test filename for test event {}".format( test_event)) test_key += ".{}.{}".format( test_event.get("test_class", ""), test_event.get("test_name", "")) # Save the most recent test event for the test key. # This allows a second test phase to overwrite the most # recent result for the test key (unique per method). # We do final reporting at the end, so we'll report based # on final results. # We do this so that a re-run caused by, perhaps, the need # to run a low-load, single-worker test run can have the final # run's results to always be used. if test_key in self.result_events: # We are replacing the result of something that was # already counted by the base class. Remove the double # counting by reducing by one the count for the test # result status. old_status = self.result_events[test_key]["status"] self.result_status_counts[old_status] -= 1 self.test_method_rerun_count += 1 if self.options.warn_on_multiple_results: print( "WARNING: test key {} already has a result: " "old:{} new:{}", self.result_events[test_key], test_event) self.result_events[test_key] = test_event else: # This is an unknown event. if self.options.assert_on_unknown_events: raise Exception("unknown event type {} from {}\n".format( event_type, test_event)) def _partition_results_by_status(self, categories): """Partitions the captured test results by event status. This permits processing test results by the category ids. @param categories the list of categories on which to partition. Follows the format described in _report_category_details(). @return a dictionary where each key is the test result status, and each entry is a list containing all the test result events that matched that test result status. Result status IDs with no matching entries will have a zero-length list. """ partitioned_events = {} for category in categories: result_status_id = category[0] matching_events = [ [key, event] for (key, event) in self.result_events.items() if event.get("status", "") == result_status_id] partitioned_events[result_status_id] = sorted( matching_events, key=lambda x: x[1]["test_name"]) return partitioned_events def _print_summary_counts(self, categories, result_events_by_status): """Prints summary counts for all categories. @param categories the list of categories on which to partition. Follows the format described in _report_category_details(). @param result_events_by_status the partitioned list of test result events in a dictionary, with the key set to the test result status id and the value set to the list of test method results that match the status id. """ # Get max length for category printed name category_with_max_printed_name = max( categories, key=lambda x: len(x[1])) max_category_name_length = len(category_with_max_printed_name[1]) banner_text = "Test Result Summary" banner_separator = "".ljust(len(banner_text), "=") self.out_file.write("\n{}\n{}\n{}\n".format( banner_separator, banner_text, banner_separator)) for category in categories: result_status_id = category[0] result_label = "{}:".format(category[1]).ljust( max_category_name_length + 1) count = len(result_events_by_status[result_status_id]) self.out_file.write("{} {:4}\n".format( result_label, count)) @classmethod def _has_printable_details(cls, categories, result_events_by_status): """Returns whether there are any test result details that need to be printed. This will spin through the results and see if any result in a category that is printable has any results to print. @param categories the list of categories on which to partition. Follows the format described in _report_category_details(). @param result_events_by_status the partitioned list of test result events in a dictionary, with the key set to the test result status id and the value set to the list of test method results that match the status id. @return True if there are any details (i.e. test results for failures, errors, unexpected successes); False otherwise. """ for category in categories: result_status_id = category[0] print_matching_tests = category[2] if print_matching_tests: if len(result_events_by_status[result_status_id]) > 0: # We found a printable details test result status # that has details to print. return True # We didn't find any test result category with printable # details. return False def _report_category_details(self, category, result_events_by_status): """Reports all test results matching the given category spec. @param category a category spec of the format [test_event_name, printed_category_name, print_matching_entries?] @param result_events_by_status the partitioned list of test result events in a dictionary, with the key set to the test result status id and the value set to the list of test method results that match the status id. """ result_status_id = category[0] print_matching_tests = category[2] detail_label = category[3] if print_matching_tests: # Sort by test name for (_, event) in result_events_by_status[result_status_id]: self.out_file.write("{}: {}.{} ({})\n".format( detail_label, event["test_class"], event["test_name"], event["test_filename"])) def _finish_output_no_lock(self): """Writes the test result report to the output file.""" self.out_file.write("\nTest Results\n") self.out_file.write( "Total Test Methods Run (excluding reruns): {}\n".format( len(self.result_events))) self.out_file.write("Test Method rerun count: {}\n".format( self.test_method_rerun_count)) # Output each of the test result entries. categories = [ # result id, printed name, print matching tests?, detail label [test_results.EventBuilder.STATUS_SUCCESS, "Success", False, None], [test_results.EventBuilder.STATUS_EXPECTED_FAILURE, "Expected Failure", False, None], [test_results.EventBuilder.STATUS_FAILURE, "Failure", True, "FAIL"], [test_results.EventBuilder.STATUS_ERROR, "Error", True, "ERROR"], [test_results.EventBuilder.STATUS_UNEXPECTED_SUCCESS, "Unexpected Success", True, "UNEXPECTED SUCCESS"], [test_results.EventBuilder.STATUS_SKIP, "Skip", False, None]] # Partition all the events by test result status result_events_by_status = self._partition_results_by_status( categories) # Print the summary self._print_summary_counts(categories, result_events_by_status) # Print the details have_details = self._has_printable_details( categories, result_events_by_status) if have_details: self.out_file.write("\nDetails:\n") for category in categories: self._report_category_details( category, result_events_by_status) def _finish_output(self): """Prepare and write the results report as all incoming events have arrived. """ with self.lock: self._finish_output_no_lock() def replaces_summary(self): return True