Changeset 74223 in webkit


Ignore:
Timestamp:
Dec 16, 2010 6:13:16 PM (13 years ago)
Author:
mihaip@chromium.org
Message:

2010-12-16 Mihai Parparita <mihaip@chromium.org>

Reviewed by Eric Seidel.

Add --exit-after-n-failures/crashes to NRWT
https://bugs.webkit.org/show_bug.cgi?id=51160

Abort test run (in a similar way to how control-C is handled) when
--exit-after-n-failures/crashes-or-timeouts are passed and we've reached
that number of unexpected failures/crashes/timeouts.

  • Scripts/webkitpy/layout_tests/port/test.py:
  • Scripts/webkitpy/layout_tests/run_webkit_tests.py:
  • Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
Location:
trunk/WebKitTools
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/WebKitTools/ChangeLog

    r74200 r74223  
     12010-12-16  Mihai Parparita  <mihaip@chromium.org>
     2
     3        Reviewed by Eric Seidel.
     4
     5        Add --exit-after-n-failures/crashes to NRWT
     6        https://bugs.webkit.org/show_bug.cgi?id=51160
     7       
     8        Abort test run (in a similar way to how control-C is handled) when
     9        --exit-after-n-failures/crashes-or-timeouts are passed and we've reached
     10        that number of unexpected failures/crashes/timeouts.
     11
     12        * Scripts/webkitpy/layout_tests/port/test.py:
     13        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
     14        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
     15
    1162010-12-16  David Levin  <levin@chromium.org>
    217
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py

    r74163 r74223  
    117117        tests.add('failures/expected/text.html',
    118118                  actual_text='text_fail-png')
     119        tests.add('failures/unexpected/crash.html', crash=True)
    119120        tests.add('failures/unexpected/text-image-checksum.html',
    120121                  actual_text='text-image-checksum_fail-txt',
    121122                  actual_checksum='text-image-checksum_fail-checksum')
     123        tests.add('failures/unexpected/timeout.html', timeout=True)
    122124        tests.add('http/tests/passes/text.html')
    123125        tests.add('http/tests/ssl/text.html')
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

    r73748 r74223  
    108108
    109109
     110class TestRunInterruptedException(Exception):
     111    """Raised when a test run should be stopped immediately."""
     112    def __init__(self, reason):
     113        self.reason = reason
     114
     115
    110116class ResultSummary(object):
    111117    """A class for partitioning the test results we get into buckets.
     
    120126        self.expected = 0
    121127        self.unexpected = 0
     128        self.unexpected_failures = 0
     129        self.unexpected_crashes_or_timeouts = 0
    122130        self.tests_by_expectation = {}
    123131        self.tests_by_timeline = {}
     
    150158            self.unexpected_results[result.filename] = result.type
    151159            self.unexpected += 1
     160            if len(result.failures):
     161                self.unexpected_failures += 1
     162            if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT:
     163                self.unexpected_crashes_or_timeouts += 1
    152164
    153165
     
    582594        """Runs the tests in the file_list.
    583595
    584         Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
    585             individual_test_timings)
    586             keyboard_interrupted is whether someone typed Ctrl^C
     596        Return: A tuple (interrupted, keyboard_interrupted, thread_timings,
     597            test_timings, individual_test_timings)
     598            interrupted is whether the run was interrupted
     599            keyboard_interrupted is whether the interruption was because someone
     600              typed Ctrl^C
    587601            thread_timings is a list of dicts with the total runtime
    588602              of each thread with 'name', 'num_tests', 'total_time' properties
     
    615629        self._printer.print_update("Starting testing ...")
    616630        keyboard_interrupted = False
     631        interrupted = False
    617632        if not self._options.dry_run:
    618633            try:
     
    622637                message_broker.cancel_workers()
    623638                keyboard_interrupted = True
     639                interrupted = True
     640            except TestRunInterruptedException, e:
     641                _log.info(e.reason)
     642                message_broker.cancel_workers()
     643                interrupted = True
    624644            except:
    625645                # Unexpected exception; don't try to clean up workers.
     
    630650            self._collect_timing_info(threads)
    631651
    632         return (keyboard_interrupted, thread_timings, test_timings,
     652        return (interrupted, keyboard_interrupted, thread_timings, test_timings,
    633653                individual_test_timings)
    634654
     
    711731        start_time = time.time()
    712732
    713         keyboard_interrupted, thread_timings, test_timings, \
     733        interrupted, keyboard_interrupted, thread_timings, test_timings, \
    714734            individual_test_timings = (
    715735            self._run_tests(self._test_files_list, result_summary))
     
    720740        retry_summary = result_summary
    721741        while (len(failures) and self._options.retry_failures and
    722             not self._retrying and not keyboard_interrupted):
     742            not self._retrying and not interrupted):
    723743            _log.info('')
    724744            _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
     
    751771
    752772        if (self._options.record_results and not self._options.dry_run and
    753             not keyboard_interrupted):
     773            not interrupted):
    754774            # Write the same data to log files and upload generated JSON files
    755775            # to appengine server.
     
    799819            self._printer.print_progress(result_summary, self._retrying,
    800820                                         self._test_files_list)
     821
     822            def interrupt_if_at_failure_limit(limit, count, message):
     823                if limit and count >= limit:
     824                    raise TestRunInterruptedException(message % count)
     825
     826            interrupt_if_at_failure_limit(
     827                self._options.exit_after_n_failures,
     828                result_summary.unexpected_failures,
     829                "Aborting run since %d failures were reached")
     830            interrupt_if_at_failure_limit(
     831                self._options.exit_after_n_crashes_or_timeouts,
     832                result_summary.unexpected_crashes_or_timeouts,
     833                "Aborting run since %d crashes or timeouts were reached")
     834
    801835
    802836    def _clobber_old_results(self):
     
    14701504        # FIXME: NRWT needs to support remote links eventually.
    14711505        _compat_shim_option("--use-remote-links-to-tests"),
    1472         # FIXME: NRWT doesn't need this option as much since failures are
    1473         # designed to be cheap.  We eventually plan to add this support.
    1474         _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
    14751506    ]
    14761507
     
    15831614            action="store_true", default=False,
    15841615            help="run all tests in parallel"),
    1585         # FIXME: Need --exit-after-n-failures N
    1586         #      Exit after the first N failures instead of running all tests
    1587         # FIXME: Need --exit-after-n-crashes N
    1588         #      Exit after the first N crashes instead of running all tests
     1616        optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
     1617            help="Exit after the first N failures instead of running all "
     1618            "tests"),
     1619        optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
     1620            nargs=1, help="Exit after the first N crashes instead of running "
     1621            "all tests"),
    15891622        # FIXME: consider: --iterations n
    15901623        #      Number of times to run the set of tests (e.g. ABCABCABC)
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

    r73654 r74223  
    333333        self._url_opened = None
    334334        res, out, err, user = logging_run(tests_included=True)
    335         self.assertEqual(res, 1)
     335        self.assertEqual(res, 3)
    336336        self.assertFalse(out.empty())
    337337        self.assertFalse(err.empty())
    338338        self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
     339
     340    def test_exit_after_n_failures(self):
     341        # Unexpected failures should result in tests stopping.
     342        tests_run = get_tests_run([
     343                'failures/unexpected/text-image-checksum.html',
     344                'passes/text.html',
     345                '--exit-after-n-failures', '1',
     346            ],
     347            tests_included=True,
     348            flatten_batches=True)
     349        self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run)
     350
     351        # But we'll keep going for expected ones.
     352        tests_run = get_tests_run([
     353                'failures/expected/text.html',
     354                'passes/text.html',
     355                '--exit-after-n-failures', '1',
     356            ],
     357            tests_included=True,
     358            flatten_batches=True)
     359        self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run)
     360
     361    def test_exit_after_n_crashes(self):
     362        # Unexpected crashes should result in tests stopping.
     363        tests_run = get_tests_run([
     364                'failures/unexpected/crash.html',
     365                'passes/text.html',
     366                '--exit-after-n-crashes-or-timeouts', '1',
     367            ],
     368            tests_included=True,
     369            flatten_batches=True)
     370        self.assertEquals(['failures/unexpected/crash.html'], tests_run)
     371
     372        # Same with timeouts.
     373        tests_run = get_tests_run([
     374                'failures/unexpected/timeout.html',
     375                'passes/text.html',
     376                '--exit-after-n-crashes-or-timeouts', '1',
     377            ],
     378            tests_included=True,
     379            flatten_batches=True)
     380        self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
     381
     382        # But we'll keep going for expected ones.
     383        tests_run = get_tests_run([
     384                'failures/expected/crash.html',
     385                'passes/text.html',
     386                '--exit-after-n-crashes-or-timeouts', '1',
     387            ],
     388            tests_included=True,
     389            flatten_batches=True)
     390        self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run)
    339391
    340392    def test_results_directory_absolute(self):
Note: See TracChangeset for help on using the changeset viewer.