Changeset 74223 in webkit
- Timestamp:
- Dec 16, 2010 6:13:16 PM (13 years ago)
- Location:
- trunk/WebKitTools
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/WebKitTools/ChangeLog
r74200 r74223 1 2010-12-16 Mihai Parparita <mihaip@chromium.org> 2 3 Reviewed by Eric Seidel. 4 5 Add --exit-after-n-failures/crashes to NRWT 6 https://bugs.webkit.org/show_bug.cgi?id=51160 7 8 Abort test run (in a similar way to how control-C is handled) when 9 --exit-after-n-failures/crashes-or-timeouts are passed and we've reached 10 that number of unexpected failures/crashes/timeouts. 11 12 * Scripts/webkitpy/layout_tests/port/test.py: 13 * Scripts/webkitpy/layout_tests/run_webkit_tests.py: 14 * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py: 15 1 16 2010-12-16 David Levin <levin@chromium.org> 2 17 -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py
r74163 r74223 117 117 tests.add('failures/expected/text.html', 118 118 actual_text='text_fail-png') 119 tests.add('failures/unexpected/crash.html', crash=True) 119 120 tests.add('failures/unexpected/text-image-checksum.html', 120 121 actual_text='text-image-checksum_fail-txt', 121 122 actual_checksum='text-image-checksum_fail-checksum') 123 tests.add('failures/unexpected/timeout.html', timeout=True) 122 124 tests.add('http/tests/passes/text.html') 123 125 tests.add('http/tests/ssl/text.html') -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
r73748 r74223 108 108 109 109 110 class TestRunInterruptedException(Exception): 111 """Raised when a test run should be stopped immediately.""" 112 def __init__(self, reason): 113 self.reason = reason 114 115 110 116 class ResultSummary(object): 111 117 """A class for partitioning the test results we get into buckets. … … 120 126 self.expected = 0 121 127 self.unexpected = 0 128 self.unexpected_failures = 0 129 self.unexpected_crashes_or_timeouts = 0 122 130 self.tests_by_expectation = {} 123 131 self.tests_by_timeline = {} … … 150 158 self.unexpected_results[result.filename] = result.type 151 159 self.unexpected += 1 160 if len(result.failures): 161 self.unexpected_failures += 1 162 if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT: 163 self.unexpected_crashes_or_timeouts += 1 152 164 153 165 … … 582 594 """Runs the tests in the file_list. 583 595 584 Return: A tuple (keyboard_interrupted, thread_timings, test_timings, 585 individual_test_timings) 586 keyboard_interrupted is whether someone typed Ctrl^C 596 Return: A tuple (interrupted, keyboard_interrupted, thread_timings, 597 test_timings, individual_test_timings) 598 interrupted is whether the run was interrupted 599 keyboard_interrupted is whether the interruption was because someone 600 typed Ctrl^C 587 601 thread_timings is a list of dicts with the total runtime 588 602 of each thread with 'name', 'num_tests', 'total_time' properties … … 615 629 self._printer.print_update("Starting testing ...") 616 630 keyboard_interrupted = False 631 interrupted = False 617 632 if not self._options.dry_run: 618 633 try: … … 622 637 message_broker.cancel_workers() 623 638 keyboard_interrupted = True 639 interrupted = True 640 except TestRunInterruptedException, e: 641 _log.info(e.reason) 642 message_broker.cancel_workers() 643 interrupted = True 624 644 except: 625 645 # Unexpected exception; don't try to clean up workers. … … 630 650 self._collect_timing_info(threads) 631 651 632 return ( keyboard_interrupted, thread_timings, test_timings,652 return (interrupted, keyboard_interrupted, thread_timings, test_timings, 633 653 individual_test_timings) 634 654 … … 711 731 start_time = time.time() 712 732 713 keyboard_interrupted, thread_timings, test_timings, \733 interrupted, keyboard_interrupted, thread_timings, test_timings, \ 714 734 individual_test_timings = ( 715 735 self._run_tests(self._test_files_list, result_summary)) … … 720 740 retry_summary = result_summary 721 741 while (len(failures) and self._options.retry_failures and 722 not self._retrying and not keyboard_interrupted):742 not self._retrying and not interrupted): 723 743 _log.info('') 724 744 _log.info("Retrying %d unexpected failure(s) ..." % len(failures)) … … 751 771 752 772 if (self._options.record_results and not self._options.dry_run and 753 not keyboard_interrupted):773 not interrupted): 754 774 # Write the same data to log files and upload generated JSON files 755 775 # to appengine server. … … 799 819 self._printer.print_progress(result_summary, self._retrying, 800 820 self._test_files_list) 821 822 def interrupt_if_at_failure_limit(limit, count, message): 823 if limit and count >= limit: 824 raise TestRunInterruptedException(message % count) 825 826 interrupt_if_at_failure_limit( 827 self._options.exit_after_n_failures, 828 result_summary.unexpected_failures, 829 "Aborting run since %d failures were reached") 830 interrupt_if_at_failure_limit( 831 self._options.exit_after_n_crashes_or_timeouts, 832 result_summary.unexpected_crashes_or_timeouts, 833 "Aborting run since %d crashes or timeouts were reached") 834 801 835 802 836 def _clobber_old_results(self): … … 1470 1504 # FIXME: NRWT needs to support remote links eventually. 1471 1505 _compat_shim_option("--use-remote-links-to-tests"), 1472 # FIXME: NRWT doesn't need this option as much since failures are1473 # designed to be cheap. We eventually plan to add this support.1474 _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),1475 1506 ] 1476 1507 … … 1583 1614 action="store_true", default=False, 1584 1615 help="run all tests in parallel"), 1585 # FIXME: Need --exit-after-n-failures N 1586 # Exit after the first N failures instead of running all tests 1587 # FIXME: Need --exit-after-n-crashes N 1588 # Exit after the first N crashes instead of running all tests 1616 optparse.make_option("--exit-after-n-failures", type="int", nargs=1, 1617 help="Exit after the first N failures instead of running all " 1618 "tests"), 1619 optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", 1620 nargs=1, help="Exit after the first N crashes instead of running " 1621 "all tests"), 1589 1622 # FIXME: consider: --iterations n 1590 1623 # Number of times to run the set of tests (e.g. ABCABCABC) -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
r73654 r74223 333 333 self._url_opened = None 334 334 res, out, err, user = logging_run(tests_included=True) 335 self.assertEqual(res, 1)335 self.assertEqual(res, 3) 336 336 self.assertFalse(out.empty()) 337 337 self.assertFalse(err.empty()) 338 338 self.assertEqual(user.url, '/tmp/layout-test-results/results.html') 339 340 def test_exit_after_n_failures(self): 341 # Unexpected failures should result in tests stopping. 342 tests_run = get_tests_run([ 343 'failures/unexpected/text-image-checksum.html', 344 'passes/text.html', 345 '--exit-after-n-failures', '1', 346 ], 347 tests_included=True, 348 flatten_batches=True) 349 self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run) 350 351 # But we'll keep going for expected ones. 352 tests_run = get_tests_run([ 353 'failures/expected/text.html', 354 'passes/text.html', 355 '--exit-after-n-failures', '1', 356 ], 357 tests_included=True, 358 flatten_batches=True) 359 self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run) 360 361 def test_exit_after_n_crashes(self): 362 # Unexpected crashes should result in tests stopping. 363 tests_run = get_tests_run([ 364 'failures/unexpected/crash.html', 365 'passes/text.html', 366 '--exit-after-n-crashes-or-timeouts', '1', 367 ], 368 tests_included=True, 369 flatten_batches=True) 370 self.assertEquals(['failures/unexpected/crash.html'], tests_run) 371 372 # Same with timeouts. 373 tests_run = get_tests_run([ 374 'failures/unexpected/timeout.html', 375 'passes/text.html', 376 '--exit-after-n-crashes-or-timeouts', '1', 377 ], 378 tests_included=True, 379 flatten_batches=True) 380 self.assertEquals(['failures/unexpected/timeout.html'], tests_run) 381 382 # But we'll keep going for expected ones. 383 tests_run = get_tests_run([ 384 'failures/expected/crash.html', 385 'passes/text.html', 386 '--exit-after-n-crashes-or-timeouts', '1', 387 ], 388 tests_included=True, 389 flatten_batches=True) 390 self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run) 339 391 340 392 def test_results_directory_absolute(self):
Note: See TracChangeset
for help on using the changeset viewer.