Changeset 81127 in webkit


Ignore:
Timestamp:
Mar 15, 2011 3:46:21 AM (13 years ago)
Author:
hayato@chromium.org
Message:

2011-03-15 Hayato Ito <​hayato@chromium.org>

Reviewed by Shinichiro Hamaji.

[NRWT] Add support for reftests to new-run-webkit-tests.

​https://bugs.webkit.org/show_bug.cgi?id=55457

  • Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py:
  • Scripts/webkitpy/layout_tests/port/base.py:
  • Scripts/webkitpy/layout_tests/port/dryrun.py:
  • Scripts/webkitpy/layout_tests/port/test.py:
  • Scripts/webkitpy/layout_tests/port/test_files.py:
  • Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
Location:
trunk/Tools
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r81123 r81127  
     12011-03-15  Hayato Ito  <hayato@chromium.org>
     2
     3        Reviewed by Shinichiro Hamaji.
     4
     5        [NRWT] Add support for reftests to new-run-webkit-tests.
     6
     7        https://bugs.webkit.org/show_bug.cgi?id=55457
     8
     9        * Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
     10        * Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
     11        * Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py:
     12        * Scripts/webkitpy/layout_tests/port/base.py:
     13        * Scripts/webkitpy/layout_tests/port/dryrun.py:
     14        * Scripts/webkitpy/layout_tests/port/test.py:
     15        * Scripts/webkitpy/layout_tests/port/test_files.py:
     16        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
     17
    1182011-03-15  Alejandro G. Castro  <alex@igalia.com>
    219

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py

    r80524 r81127  
    4545
    4646
    47 class ExpectedDriverOutput:
    48     """Groups information about an expected driver output."""
    49     def __init__(self, text, image, image_hash):
    50         self.text = text
    51         self.image = image
    52         self.image_hash = image_hash
    53 
    54 
    5547class SingleTestRunner:
    5648
    … …  
    6456        self._testname = port.relative_test_filename(test_input.filename)
    6557
     58        self._is_reftest = False
     59        self._is_mismatch_reftest = False
     60        self._reference_filename = None
     61
     62        fs = port._filesystem
     63        reftest_expected_filename = port.reftest_expected_filename(self._filename)
     64        if fs.exists(reftest_expected_filename):
     65            self._is_reftest = True
     66            self._reference_filename = reftest_expected_filename
     67
     68        reftest_expected_mismatch_filename = port.reftest_expected_mismatch_filename(self._filename)
     69        if fs.exists(reftest_expected_mismatch_filename):
     70            if self._is_reftest:
     71                _log.error('It is not allowed that one test file has both'
     72                           ' expected.html file and expected-mismatch.html file'
     73                           ' at the same time. Please remove either %s or %s.',
     74                           reftest_expected_filename, reftest_expected_mismatch_filename)
     75            else:
     76                self._is_reftest = True
     77                self._is_mismatch_reftest = True
     78                self._reference_filename = reftest_expected_mismatch_filename
     79
     80        if self._is_reftest:
     81            # Detect and report a test which has a wrong combination of expectation files.
     82            # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
     83            # 'foo-expected.txt', we should warn users. One test file must be used exclusively
     84            # in either layout tests or reftests, but not in both.
     85            for suffix in ['.txt', '.checksum', '.png']:
     86                expected_filename = self._port.expected_filename(self._filename, suffix)
     87                if fs.exists(expected_filename):
     88                    _log.error('The reftest (%s) can not have an expectation file (%s).'
     89                               ' Please remove that file.', self._testname, expected_filename)
     90
    6691    def _expected_driver_output(self):
    67         return ExpectedDriverOutput(self._port.expected_text(self._filename),
    68                                     self._port.expected_image(self._filename),
    69                                     self._port.expected_checksum(self._filename))
     92        return base.DriverOutput(self._port.expected_text(self._filename),
     93                                 self._port.expected_image(self._filename),
     94                                 self._port.expected_checksum(self._filename))
    7095
    7196    def _should_fetch_expected_checksum(self):
    … …  
    85110    def run(self):
    86111        if self._options.new_baseline or self._options.reset_results:
    87             return self._run_rebaseline()
     112            if self._is_reftest:
     113                # Returns a dummy TestResult. We don't have to rebase for reftests.
     114                return TestResult(self._filename)
     115            else:
     116                return self._run_rebaseline()
     117        if self._is_reftest:
     118            return self._run_reftest()
    88119        return self._run_compare_test()
    89120
    … …  
    99130        driver_output = self._driver.run_test(self._driver_input())
    100131        failures = self._handle_error(driver_output)
     132        test_result_writer.write_test_result(self._port, self._options.results_directory, self._filename,
     133                                             driver_output, None, failures)
    101134        # FIXME: It the test crashed or timed out, it might be bettter to avoid
    102135        # to write new baselines.
    … …  
    146179        port.update_baseline(output_path, data)
    147180
    148     def _handle_error(self, driver_output):
     181    def _handle_error(self, driver_output, reference_filename=None):
     182        """Returns test failures if some unusual errors happen in driver's run.
     183
     184        Args:
     185          driver_output: The output from the driver.
     186          reference_filename: The full path to the reference file which produced the driver_output.
     187              This arg is optional and should be used only in reftests until we have a better way to know
     188              which html file is used for producing the driver_output.
     189        """
    149190        failures = []
    150191        fs = self._port._filesystem
    151192        if driver_output.timeout:
    152             failures.append(test_failures.FailureTimeout())
     193            failures.append(test_failures.FailureTimeout(reference_filename))
     194
     195        if reference_filename:
     196            testname = self._port.relative_test_filename(reference_filename)
     197        else:
     198            testname = self._testname
     199
    153200        if driver_output.crash:
    154             failures.append(test_failures.FailureCrash())
    155             _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname,
     201            failures.append(test_failures.FailureCrash(reference_filename))
     202            _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, testname,
    156203                                                      driver_output.error))
    157             # FIXME: Use test_result_writer module.
    158             stack_filename = fs.join(self._options.results_directory, self._testname)
    159             stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt"
    160             fs.maybe_make_directory(fs.dirname(stack_filename))
    161             fs.write_text_file(stack_filename, driver_output.error)
    162204        elif driver_output.error:
    163             _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname,
     205            _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, testname,
    164206                                                           driver_output.error))
    165207        return failures
    … …  
    211253            failures.append(test_failures.FailureImageHashMismatch())
    212254        return failures
     255
     256    def _run_reftest(self):
     257        driver_output1 = self._driver.run_test(self._driver_input())
     258        driver_output2 = self._driver.run_test(
     259            base.DriverInput(self._reference_filename, self._timeout, driver_output1.image_hash))
     260        test_result = self._compare_output_with_reference(driver_output1, driver_output2)
     261
     262        test_result_writer.write_test_result(self._port, self._options.results_directory, self._filename,
     263                                             driver_output1, driver_output2, test_result.failures)
     264        return test_result
     265
     266    def _compare_output_with_reference(self, driver_output1, driver_output2):
     267        total_test_time = driver_output1.test_time + driver_output2.test_time
     268        failures = []
     269        failures.extend(self._handle_error(driver_output1))
     270        if failures:
     271            # Don't continue any more if we already have crash or timeout.
     272            return TestResult(self._filename, failures, total_test_time)
     273        failures.extend(self._handle_error(driver_output2, reference_filename=self._reference_filename))
     274        if failures:
     275            return TestResult(self._filename, failures, total_test_time)
     276
     277        if self._is_mismatch_reftest:
     278            if driver_output1.image_hash == driver_output2.image_hash:
     279                failures.append(test_failures.FailureReftestMismatchDidNotOccur())
     280        elif driver_output1.image_hash != driver_output2.image_hash:
     281            failures.append(test_failures.FailureReftestMismatch())
     282        return TestResult(self._filename, failures, total_test_time)

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py

    r80558 r81127  
    6161        is_image_failure = (FailureImageHashIncorrect in failure_types or
    6262                            FailureImageHashMismatch in failure_types)
     63        is_reftest_failure = (FailureReftestMismatch in failure_types or
     64                              FailureReftestMismatchDidNotOccur in failure_types)
    6365        if is_text_failure and is_image_failure:
    6466            return test_expectations.IMAGE_PLUS_TEXT
    6567        elif is_text_failure:
    6668            return test_expectations.TEXT
    67         elif is_image_failure:
     69        elif is_image_failure or is_reftest_failure:
    6870            return test_expectations.IMAGE
    6971        else:
    … …  
    178180    happens."""
    179181
     182    def __init__(self, reference_filename=None):
     183        self.reference_filename = reference_filename
     184
    180185    @staticmethod
    181186    def message():
    … …  
    183188
    184189    def result_html_output(self, filename):
     190        if self.reference_filename:
     191            return "<strong>%s</strong> (occured in <a href=%s>expected html</a>)" % (
     192                self.message(), self.reference_filename)
    185193        return "<strong>%s</strong>" % self.message()
    186194
    … …  
    192200    """DumpRenderTree crashed."""
    193201
     202    def __init__(self, reference_filename=None):
     203        self.reference_filename = reference_filename
     204
    194205    @staticmethod
    195206    def message():
    … …  
    199210        # FIXME: create a link to the minidump file
    200211        stack = self.relative_output_filename(filename, "-stack.txt")
    201         return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(),
    202                                                              stack)
     212        if self.reference_filename:
     213            return "<strong>%s</strong> <a href=%s>stack</a> (occured in <a href=%s>expected html</a>)" % (
     214                self.message(), stack, self.reference_filename)
     215        else:
     216            return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(), stack)
    203217
    204218    def should_kill_dump_render_tree(self):
    … …  
    280294        return "<strong>%s</strong>" % self.message()
    281295
     296
     297class FailureReftestMismatch(ComparisonTestFailure):
     298    """The result didn't match the reference rendering."""
     299
     300    OUT_FILENAMES = ("-expected.html", "-expected.png", "-actual.png",
     301                     "-diff.png",)
     302
     303    @staticmethod
     304    def message():
     305        return "Mismatch with reference"
     306
     307    def output_links(self, filename, out_names):
     308        links = ['']
     309        uris = [self.relative_output_filename(filename, output_filename)
     310                for output_filename in out_names]
     311        for text, uri in zip(['-expected.html', 'expected', 'actual', 'diff'], uris):
     312            links.append("<a href='%s'>%s</a>" % (uri, text))
     313        return ' '.join(links)
     314
     315
     316class FailureReftestMismatchDidNotOccur(ComparisonTestFailure):
     317    """Unexpected match between the result and the reference rendering."""
     318
     319    OUT_FILENAMES = ("-expected-mismatch.html", "-actual.png",)
     320
     321    @staticmethod
     322    def message():
     323        return "Mismatch with the reference did not occur"
     324
     325    def output_links(self, filename, out_names):
     326        links = ['']
     327        uris = [self.relative_output_filename(filename, output_filename)
     328                for output_filename in out_names]
     329        for text, uri in zip(['-expected-mismatch.html', 'image'], uris):
     330            links.append("<a href='%s'>%s</a>" % (uri, text))
     331        return ' '.join(links)
     332
     333
    282334# Convenient collection of all failure classes for anything that might
    283335# need to enumerate over them all.
    … …  
    285337                       FailureTextMismatch, FailureMissingImageHash,
    286338                       FailureMissingImage, FailureImageHashMismatch,
    287                        FailureImageHashIncorrect)
     339                       FailureImageHashIncorrect, FailureReftestMismatch,
     340                       FailureReftestMismatchDidNotOccur)

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py

    r78961 r81127  
    6565                imagehash_mismatch_failure = failure
    6666        elif isinstance(failure, test_failures.FailureCrash):
    67             writer.write_crash_report(driver_output.error)
     67            if failure.reference_filename:
     68                writer.write_crash_report(expected_driver_output.error)
     69            else:
     70                writer.write_crash_report(driver_output.error)
     71        elif isinstance(failure, test_failures.FailureReftestMismatch):
     72            writer.write_image_files(driver_output.image, expected_driver_output.image)
     73            writer.create_image_diff_and_write_result(driver_output.image, expected_driver_output.image)
     74            writer.copy_file(port.reftest_expected_filename(filename), '-expected.html')
     75        elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
     76            writer.write_image_files(driver_output.image, expected_image=None)
     77            writer.copy_file(port.reftest_expected_mismatch_filename(filename), '-expected-mismatch.html')
    6878        else:
    6979            assert isinstance(failure, (test_failures.FailureTimeout,))
    … …  
    136146
    137147        fs = self._port._filesystem
    138         if output:
     148        if output is not None:
    139149            fs.write_binary_file(actual_filename, output)
    140         if expected:
     150        if expected is not None:
    141151            fs.write_binary_file(expected_filename, expected)
    142152
    … …  
    194204        diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
    195205        return self._port.diff_image(actual_image, expected_image, diff_filename)
     206
     207    def copy_file(self, src_filepath, dst_extension):
     208        fs = self._port._filesystem
     209        assert fs.exists(src_filepath), 'src_filepath: %s' % src_filepath
     210        dst_filename = self.output_filename(dst_extension)
     211        self._make_output_directory()
     212        fs.copyfile(src_filepath, dst_filename)

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py

    r80549 r81127  
    348348        return text.replace("\r\n", "\n")
    349349
     350    def reftest_expected_filename(self, filename):
     351        """Return the filename of reference we expect the test matches."""
     352        return self.expected_filename(filename, '.html')
     353
     354    def reftest_expected_mismatch_filename(self, filename):
     355        """Return the filename of reference we don't expect the test matches."""
     356        return self.expected_filename(filename, '-mismatch.html')
     357
    350358    def filename_to_uri(self, filename):
    351359        """Convert a test file (which is an absolute path) to a URI."""

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py

    r77603 r81127  
    119119    def run_test(self, driver_input):
    120120        start_time = time.time()
    121         text_output = self._port.expected_text(driver_input.filename)
    122 
    123         if driver_input.image_hash is not None:
     121        fs = self._port._filesystem
     122        if fs.exists(self._port.reftest_expected_filename(driver_input.filename)) or \
     123            fs.exists(self._port.reftest_expected_mismatch_filename(driver_input.filename)):
     124            text_output = 'test-text'
     125            image = 'test-image'
     126            hash = 'test-checksum'
     127        elif driver_input.filename.endswith('-expected.html'):
     128            text_output = 'test-text'
     129            image = 'test-image'
     130            hash = 'test-checksum'
     131        elif driver_input.filename.endswith('-expected-mismatch.html'):
     132            text_output = 'test-text-mismatch'
     133            image = 'test-image-mismatch'
     134            hash = 'test-checksum-mismatch'
     135        elif driver_input.image_hash is not None:
     136            text_output = self._port.expected_text(driver_input.filename)
    124137            image = self._port.expected_image(driver_input.filename)
    125138            hash = self._port.expected_checksum(driver_input.filename)
    126139        else:
     140            text_output = self._port.expected_text(driver_input.filename)
    127141            image = None
    128142            hash = None

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py

    r78522 r81127  
    5151        self.error = ''
    5252        self.timeout = False
     53        self.is_reftest = False
    5354
    5455        # The values of each field are treated as raw byte strings. They
    … …  
    7879            test.__dict__[key] = value
    7980        self.tests[name] = test
     81
     82    def add_reftest(self, name, reference_name, same_image):
     83        self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
     84        if same_image:
     85            self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
     86        else:
     87            self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
    8088
    8189    def keys(self):
    … …  
    130138    tests.add('passes/text.html',
    131139              expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
     140
     141    # For reftests.
     142    tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
     143    tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
     144    tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
     145    tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
     146    tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
     147    tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
     148    # FIXME: Add a reftest which crashes.
     149
    132150    tests.add('websocket/tests/passes/text.html')
    133151    return tests
    … …  
    159177    for test in test_list.tests.values():
    160178        add_file(files, test, '.html', '')
     179        if test.is_reftest:
     180            continue
    161181        add_file(files, test, '-expected.txt', test.expected_text)
    162182        add_file(files, test, '-expected.checksum', test.expected_checksum)
    … …  
    170190WONTFIX : failures/expected/image.html = PASS
    171191WONTFIX : failures/expected/image_checksum.html = IMAGE
     192WONTFIX : failures/expected/mismatch.html = IMAGE
    172193WONTFIX : failures/expected/missing_check.html = MISSING PASS
    173194WONTFIX : failures/expected/missing_image.html = MISSING PASS
    … …  
    176197WONTFIX : failures/expected/newlines_trailing.html = TEXT
    177198WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
     199WONTFIX : failures/expected/reftest.html = IMAGE
    178200WONTFIX : failures/expected/text.html = TEXT
    179201WONTFIX : failures/expected/timeout.html = TIMEOUT

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/test_files.py

    r77556 r81127  
    106106
    107107
    108 def _is_reference_html_file(filename):
     108def is_reference_html_file(filename):
    109109    """Return true if the filename points to a reference HTML file."""
    110110    if (filename.endswith('-expected.html') or
    111111        filename.endswith('-expected-mismatch.html')):
    112         _log.warn("Reftests are not supported - ignoring %s" % filename)
    113112        return True
    114113    return False
    … …  
    118117    """Return true if the filename points to a test file."""
    119118    return (_has_supported_extension(filesystem, filename) and
    120             not _is_reference_html_file(filename))
     119            not is_reference_html_file(filename))

  • trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

    r80284 r81127  
    5656from webkitpy.layout_tests.layout_package import dump_render_tree_thread
    5757from webkitpy.layout_tests.port.test import TestPort, TestDriver
     58from webkitpy.layout_tests.port.test_files import is_reference_html_file
    5859from webkitpy.python24.versioning import compare_version
    5960from webkitpy.test.skip import skip_if
    … …  
    125126
    126127
    127 def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False, filesystem=None):
     128def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
     129                  filesystem=None, include_reference_html=False):
    128130    extra_args = extra_args or []
    129131    if not tests_included:
    … …  
    137139    test_batches = []
    138140
     141
    139142    class RecordingTestDriver(TestDriver):
    140143        def __init__(self, port, worker_number):
    … …  
    154157                test_batches.append(self._current_test_batch)
    155158            test_name = self._port.relative_test_filename(test_input.filename)
    156             self._current_test_batch.append(test_name)
     159            # In case of reftest, one test calls the driver's run_test() twice.
     160            # We should not add a reference html used by reftests to tests unless include_reference_html parameter
     161            # is explicitly given.
     162            if include_reference_html or not is_reference_html_file(test_input.filename):
     163                self._current_test_batch.append(test_name)
    157164            return TestDriver.run_test(self, test_input)
    158165
    … …  
    353360        self._url_opened = None
    354361        res, out, err, user = logging_run(tests_included=True)
    355         self.assertEqual(res, 3)
     362
     363        # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
     364        # FIXME: It's nice to have a routine in port/test.py that returns this number.
     365        unexpected_tests_count = 5
     366
     367        self.assertEqual(res, unexpected_tests_count)
    356368        self.assertFalse(out.empty())
    357369        self.assertFalse(err.empty())
    … …  
    516528        self.assertRaises(ValueError, logging_run,
    517529                          ['--worker-model', 'unknown'])
     530
     531    def test_reftest_run(self):
     532        tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True)
     533        self.assertEquals(['passes/reftest.html'], tests_run)
     534
     535    def test_reftest_expected_html_should_be_ignored(self):
     536        tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True)
     537        self.assertEquals([], tests_run)
     538
     539    def test_reftest_driver_should_run_expected_html(self):
     540        tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True,
     541                                  include_reference_html=True)
     542        self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run)
     543
     544    def test_reftest_driver_should_run_expected_mismatch_html(self):
     545        tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True,
     546                                  include_reference_html=True)
     547        self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run)
     548
    518549
    519550MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
Note: See TracChangeset for help on using the changeset viewer.