Changeset 101907 in webkit


Ignore:
Timestamp:
Dec 2, 2011 10:06:56 PM (12 years ago)
Author:
tony@chromium.org
Message:

[NRWT] reftest asserts intermittently on the Qt-WK2 bot
https://bugs.webkit.org/show_bug.cgi?id=73453

Reviewed by Ryosuke Niwa.

We used to assert if a ref test didn't generate pixel results. Instead, just claim it is a failing test.
The verbose output will include the failure reason.

  • Scripts/webkitpy/layout_tests/controllers/single_test_runner.py: Don't assert, return FailureReftestNoImagesGenerated instead.
  • Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:

(write_test_result):

  • Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py: Remove stray letter t.
  • Scripts/webkitpy/layout_tests/models/test_failures.py: Add a new failure type: FailureReftestNoImagesGenerated

(determine_result_type):
(FailureReftestNoImagesGenerated.init):
(FailureReftestNoImagesGenerated):
(FailureReftestNoImagesGenerated.message):

  • Scripts/webkitpy/layout_tests/port/test.py: Add a test case for integration tests.
  • Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py: Update test counts.

(MainTest.test_run_singly_actually_runs_tests):
(MainTest.test_unexpected_failures):
(EndToEndTest.test_end_to_end):

Location:
trunk/Tools
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • trunk/Tools/ChangeLog

    r101901 r101907  
     12011-12-02  Tony Chang  <tony@chromium.org>
     2
     3        [NRWT] reftest asserts intermittently on the Qt-WK2 bot
     4        https://bugs.webkit.org/show_bug.cgi?id=73453
     5
     6        Reviewed by Ryosuke Niwa.
     7
     8        We used to assert if a ref test didn't generate pixel results. Instead, just claim it is a failing test.
     9        The verbose output will include the failure reason.
     10
     11        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py: Don't assert, return FailureReftestNoImagesGenerated instead.
     12        * Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
     13        (write_test_result):
     14        * Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py: Remove stray letter t.
     15        * Scripts/webkitpy/layout_tests/models/test_failures.py: Add a new failure type: FailureReftestNoImagesGenerated
     16        (determine_result_type):
     17        (FailureReftestNoImagesGenerated.__init__):
     18        (FailureReftestNoImagesGenerated):
     19        (FailureReftestNoImagesGenerated.message):
     20        * Scripts/webkitpy/layout_tests/port/test.py: Add a test case for integration tests.
     21        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py: Update test counts.
     22        (MainTest.test_run_singly_actually_runs_tests):
     23        (MainTest.test_unexpected_failures):
     24        (EndToEndTest.test_end_to_end):
     25
    1262011-12-02  Ryosuke Niwa  <rniwa@webkit.org>
    227
  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

    r101845 r101907  
    297297            return TestResult(self._test_name, failures, total_test_time, has_stderr)
    298298
    299         assert(driver_output1.image_hash or driver_output2.image_hash)
    300 
    301         if mismatch:
     299        if not driver_output1.image_hash and not driver_output2.image_hash:
     300            failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
     301        elif mismatch:
    302302            if driver_output1.image_hash == driver_output2.image_hash:
    303303                failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename))
  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py

    r101739 r101907  
    8181            writer.copy_file(failure.reference_filename)
    8282        else:
    83             assert isinstance(failure, (test_failures.FailureTimeout,))
     83            assert isinstance(failure, (test_failures.FailureTimeout, test_failures.FailureReftestNoImagesGenerated))
    8484
    8585
  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer_unittest.py

    r101845 r101907  
    3535class TestResultWriterTest(unittest.TestCase):
    3636
    37     def test_reftest_diff_imaget(self):
     37    def test_reftest_diff_image(self):
    3838        """A write_test_result should call port.diff_image with tolerance=0 in case of FailureReftestMismatch."""
    3939        used_tolerance_values = []
  • trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py

    r99651 r101907  
    6060        is_image_failure = (FailureImageHashIncorrect in failure_types or
    6161                            FailureImageHashMismatch in failure_types)
    62         is_reftest_failure = (FailureReftestMismatch in failure_types or
    63                               FailureReftestMismatchDidNotOccur in failure_types)
     62        is_reftest_failure = set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
    6463        is_audio_failure = (FailureAudioMismatch in failure_types)
    6564        if is_text_failure and is_image_failure:
     
    206205
    207206
     207class FailureReftestNoImagesGenerated(TestFailure):
     208    """Both the reftest and the -expected html file didn't generate pixel results."""
     209
     210    def __init__(self, reference_filename=None):
     211        self.reference_filename = reference_filename
     212
     213    @staticmethod
     214    def message():
     215        return "Reftest didn't generate pixel results."
     216
     217
    208218class FailureMissingAudio(TestFailure):
    209219    """Actual result image was missing."""
     
    228238                       FailureMissingImage, FailureImageHashMismatch,
    229239                       FailureImageHashIncorrect, FailureReftestMismatch,
    230                        FailureReftestMismatchDidNotOccur)
     240                       FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)
  • trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py

    r101845 r101907  
    187187    tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
    188188    tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
     189    tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
     190    tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
    189191    # FIXME: Add a reftest which crashes.
    190192    tests.add('reftests/foo/test.html')
  • trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

    r101845 r101907  
    188188# Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
    189189# FIXME: It's nice to have a routine in port/test.py that returns this number.
    190 unexpected_tests_count = 11
     190unexpected_tests_count = 12
    191191
    192192
     
    384384    def test_run_singly_actually_runs_tests(self):
    385385        res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
    386         self.assertEquals(res, 7)
     386        self.assertEquals(res, 8)
    387387
    388388    def test_single_file(self):
Note: See TracChangeset for help on using the changeset viewer.