Changeset 90413 in webkit
- Timestamp:
- Jul 5, 2011 4:16:18 PM (13 years ago)
- Location:
- trunk
- Files:
-
- 7 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/LayoutTests/ChangeLog
r90405 r90413 1 2011-07-05 Eric Seidel <eric@webkit.org> 2 3 buildbot needs to understand whether NRWT exited early after having too many failures 4 https://bugs.webkit.org/show_bug.cgi?id=63839 5 6 Reviewed by Adam Barth. 7 8 Teach the new results.html how to display a warning when testing exited early. 9 The warning isn't quite as nice as ORWT since I couldn't figure out how to 10 find the total tests run, or total unexpected crashes. I figure 11 this is enough to get us going and we can refine it further. 12 13 * fast/harness/results.html: 14 1 15 2011-07-05 Sam Weinig <sam@webkit.org> 2 16 -
trunk/LayoutTests/fast/harness/results.html
r89873 r90413 75 75 top: -0.3em; 76 76 left: 1px; 77 } 78 79 .stopped-running-early-message { 80 border: 3px solid #d00; 81 font-weight: bold; 82 display: inline-block; 83 padding: 3px; 77 84 } 78 85 … … 1017 1024 '</div></div>'; 1018 1025 1026 if (globalState().results.interrupted) 1027 html += "<p class='stopped-running-early-message'>Testing exited early.</p>" 1028 1019 1029 html += failingTestsTable(globalState().nonFlakyFailingTests, 1020 1030 'Tests where results did not match expected results', 'results-table'); -
trunk/Tools/ChangeLog
r90412 r90413 1 2011-07-05 Eric Seidel <eric@webkit.org> 2 3 buildbot needs to understand whether NRWT exited early after having too many failures 4 https://bugs.webkit.org/show_bug.cgi?id=63839 5 6 Reviewed by Adam Barth. 7 8 Fix-up the exited early messages printed by NRWT so that 9 the buildbot can parse them as expected. 10 It looks for lines using "if line.find('Exiting early') >= 0:" 11 12 I also plumbed the "early exit" status through to results.json 13 in the form of an "interrupted" bool. It was unclear to me 14 if results.json already had enough information to compute this bool 15 itself. It's possible Ojan could come up with a better fix. 16 17 * Scripts/webkitpy/layout_tests/layout_package/manager.py: 18 * Scripts/webkitpy/layout_tests/layout_package/manager_unittest.py: 19 * Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py: 20 * Scripts/webkitpy/layout_tests/layout_package/result_summary.py: 21 1 22 2011-07-05 Eric Seidel <eric@webkit.org> 2 23 -
trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py
r90285 r90413 74 74 75 75 76 def summarize_results(port_obj, expectations, result_summary, retry_summary, test_timings, only_unexpected): 76 # FIXME: This should be on the Manager class (since that's the only caller) 77 # or split off from Manager onto another helper class, but should not be a free function. 78 # Most likely this should be made into its own class, and this super-long function 79 # split into many helper functions. 80 def summarize_results(port_obj, expectations, result_summary, retry_summary, test_timings, only_unexpected, interrupted): 77 81 """Summarize failing results as a dict. 78 82 … … 102 106 tbe = result_summary.tests_by_expectation 103 107 tbt = result_summary.tests_by_timeline 104 results['fixable'] = len(tbt[test_expectations.NOW] - 105 tbe[test_expectations.PASS]) 106 results['skipped'] = len(tbt[test_expectations.NOW] & 107 tbe[test_expectations.SKIP]) 108 results['fixable'] = len(tbt[test_expectations.NOW] - tbe[test_expectations.PASS]) 109 results['skipped'] = len(tbt[test_expectations.NOW] & tbe[test_expectations.SKIP]) 108 110 109 111 num_passes = 0 … … 208 210 results['num_regressions'] = num_regressions 209 211 results['uses_expectations_file'] = port_obj.uses_test_expectations_file() 212 results['interrupted'] = interrupted # Does results.html have enough information to compute this itself? (by checking total number of results vs. total number of tests?) 210 213 results['layout_tests_dir'] = port_obj.layout_tests_dir() 211 214 results['has_wdiff'] = port_obj.wdiff_available() … … 224 227 def __init__(self, reason): 225 228 self.reason = reason 229 self.msg = reason 226 230 227 231 def __reduce__(self): … … 816 820 result_summary.unexpected) 817 821 818 unexpected_results = summarize_results(self._port, 819 self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=True) 822 unexpected_results = summarize_results(self._port, self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=True, interrupted=interrupted) 820 823 self._printer.print_unexpected_results(unexpected_results) 821 824 … … 830 833 # Write the same data to log files and upload generated JSON files 831 834 # to appengine server. 832 summarized_results = summarize_results(self._port, 833 self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=False) 834 self._upload_json_files(unexpected_results, summarized_results, result_summary, 835 individual_test_timings) 835 summarized_results = summarize_results(self._port, self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=False, interrupted=interrupted) 836 self._upload_json_files(unexpected_results, summarized_results, result_summary, individual_test_timings) 836 837 837 838 # Write the summary to disk (results.html) and display it if requested. … … 885 886 self._update_summary_with_result(result_summary, result) 886 887 888 def _interrupt_if_at_failure_limits(self, result_summary): 889 # Note: The messages in this method are constructed to match old-run-webkit-tests 890 # so that existing buildbot grep rules work. 891 def interrupt_if_at_failure_limit(limit, failure_count, result_summary, message): 892 if limit and failure_count >= limit: 893 message += " %d tests run." % (result_summary.expected + result_summary.unexpected) 894 raise TestRunInterruptedException(message) 895 896 interrupt_if_at_failure_limit( 897 self._options.exit_after_n_failures, 898 result_summary.unexpected_failures, 899 result_summary, 900 "Exiting early after %d failures." % result_summary.unexpected_failures) 901 interrupt_if_at_failure_limit( 902 self._options.exit_after_n_crashes_or_timeouts, 903 result_summary.unexpected_crashes + result_summary.unexpected_timeouts, 904 result_summary, 905 # This differs from ORWT because it does not include WebProcess crashes. 906 "Exiting early after %d crashes and %d timeouts." % (result_summary.unexpected_crashes, result_summary.unexpected_timeouts)) 907 887 908 def _update_summary_with_result(self, result_summary, result): 888 909 if result.type == test_expectations.SKIP: 889 910 result_summary.add(result, expected=True) 890 911 else: 891 expected = self._expectations.matches_an_expected_result( 892 result.filename, result.type, self._options.pixel_tests) 912 expected = self._expectations.matches_an_expected_result(result.filename, result.type, self._options.pixel_tests) 893 913 result_summary.add(result, expected) 894 exp_str = self._expectations.get_expectations_string( 895 result.filename) 914 exp_str = self._expectations.get_expectations_string(result.filename) 896 915 got_str = self._expectations.expectation_to_string(result.type) 897 916 self._printer.print_test_result(result, expected, exp_str, got_str) 898 self._printer.print_progress(result_summary, self._retrying, 899 self._test_files_list) 900 901 def interrupt_if_at_failure_limit(limit, count, message): 902 if limit and count >= limit: 903 raise TestRunInterruptedException(message % count) 904 905 interrupt_if_at_failure_limit( 906 self._options.exit_after_n_failures, 907 result_summary.unexpected_failures, 908 "Aborting run since %d failures were reached") 909 interrupt_if_at_failure_limit( 910 self._options.exit_after_n_crashes_or_timeouts, 911 result_summary.unexpected_crashes_or_timeouts, 912 "Aborting run since %d crashes or timeouts were reached") 917 self._printer.print_progress(result_summary, self._retrying, self._test_files_list) 918 self._interrupt_if_at_failure_limits(result_summary) 913 919 914 920 def _clobber_old_results(self): … … 1370 1376 1371 1377 1378 # FIXME: These two free functions belong either on manager (since it's the only one 1379 # which uses them) or in a different file (if they need to be re-used). 1372 1380 def path_key(filesystem, path): 1373 1381 """Turns a path into a list with two sublists, the natural key of the -
trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_unittest.py
r90211 r90413 36 36 from webkitpy.thirdparty.mock import Mock 37 37 38 from webkitpy.layout_tests.layout_package import manager 38 from webkitpy.layout_tests.layout_package.manager import Manager, natural_sort_key, path_key, TestRunInterruptedException 39 from webkitpy.layout_tests.layout_package.result_summary import ResultSummary 40 from webkitpy.tool.mocktool import MockOptions 39 41 40 42 41 class ManagerWrapper( manager.Manager):43 class ManagerWrapper(Manager): 42 44 def _get_test_input_for_file(self, test_file): 43 45 return test_file … … 80 82 self.assertEqual(expected_tests_to_http_lock, set(multi_thread_results[0][1])) 81 83 84 def test_interrupt_if_at_failure_limits(self): 85 port = Mock() 86 port._filesystem = filesystem_mock.MockFileSystem() 87 manager = Manager(port=port, options=MockOptions(), printer=Mock()) 88 89 manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None) 90 result_summary = ResultSummary(expectations=Mock(), test_files=[]) 91 result_summary.unexpected_failures = 100 92 result_summary.unexpected_crashes = 50 93 result_summary.unexpected_timeouts = 50 94 # No exception when the exit_after* options are None. 95 manager._interrupt_if_at_failure_limits(result_summary) 96 97 # No exception when we haven't hit the limit yet. 98 manager._options.exit_after_n_failures = 101 99 manager._options.exit_after_n_crashes_or_timeouts = 101 100 manager._interrupt_if_at_failure_limits(result_summary) 101 102 # Interrupt if we've exceeded either limit: 103 manager._options.exit_after_n_crashes_or_timeouts = 10 104 self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary) 105 106 manager._options.exit_after_n_crashes_or_timeouts = None 107 manager._options.exit_after_n_failures = 10 108 exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary) 109 82 110 83 111 class NaturalCompareTest(unittest.TestCase): 84 112 def assert_cmp(self, x, y, result): 85 self.assertEquals(cmp( manager.natural_sort_key(x), manager.natural_sort_key(y)), result)113 self.assertEquals(cmp(natural_sort_key(x), natural_sort_key(y)), result) 86 114 87 115 def test_natural_compare(self): … … 108 136 109 137 def path_key(self, k): 110 return manager.path_key(self.filesystem, k)138 return path_key(self.filesystem, k) 111 139 112 140 def assert_cmp(self, x, y, result): -
trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py
r87967 r90413 446 446 paths, rs, exp = self.get_result_summary(tests, expectations) 447 447 if expected: 448 rs.add(self.get_result('passes/text.html', test_expectations.PASS), 449 expected) 450 rs.add(self.get_result('failures/expected/timeout.html', 451 test_expectations.TIMEOUT), expected) 452 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), 453 expected) 448 rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected) 449 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected) 450 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected) 454 451 elif passing: 455 452 rs.add(self.get_result('passes/text.html'), expected) … … 457 454 rs.add(self.get_result('failures/expected/crash.html'), expected) 458 455 else: 459 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), 460 expected) 461 rs.add(self.get_result('failures/expected/timeout.html', 462 test_expectations.CRASH), expected) 463 rs.add(self.get_result('failures/expected/crash.html', 464 test_expectations.TIMEOUT), 465 expected) 456 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected) 457 rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected) 458 rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected) 466 459 retry = rs 467 460 if flaky: 468 paths, retry, exp = self.get_result_summary(tests, 469 expectations) 461 paths, retry, exp = self.get_result_summary(tests, expectations) 470 462 retry.add(self.get_result('passes/text.html'), True) 471 463 retry.add(self.get_result('failures/expected/timeout.html'), True) 472 464 retry.add(self.get_result('failures/expected/crash.html'), True) 473 unexpected_results = manager.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True )465 unexpected_results = manager.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False) 474 466 return unexpected_results 475 467 476 tests = ['passes/text.html', 'failures/expected/timeout.html', 477 'failures/expected/crash.html'] 468 tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html'] 478 469 expectations = '' 479 470 … … 484 475 self.assertTrue(out.empty()) 485 476 486 printer, err, out = self.get_printer(['--print', 487 'unexpected-results']) 477 printer, err, out = self.get_printer(['--print', 'unexpected-results']) 488 478 489 479 # test everything running as expected -
trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py
r85875 r90413 53 53 self.unexpected = 0 54 54 self.unexpected_failures = 0 55 self.unexpected_crashes_or_timeouts = 0 55 self.unexpected_crashes = 0 56 self.unexpected_timeouts = 0 56 57 self.tests_by_expectation = {} 57 58 self.tests_by_timeline = {} … … 63 64 self.tests_by_expectation[expectation] = set() 64 65 for timeline in TestExpectations.TIMELINES.values(): 65 self.tests_by_timeline[timeline] = ( 66 expectations.get_tests_with_timeline(timeline)) 66 self.tests_by_timeline[timeline] = expectations.get_tests_with_timeline(timeline) 67 67 68 68 def add(self, result, expected): … … 86 86 if len(result.failures): 87 87 self.unexpected_failures += 1 88 if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT: 89 self.unexpected_crashes_or_timeouts += 1 88 if result.type == test_expectations.CRASH: 89 self.unexpected_crashes += 1 90 elif result.type == test_expectations.TIMEOUT: 91 self.unexpected_timeouts += 1
Note: See TracChangeset
for help on using the changeset viewer.