Changeset 129148 in webkit


Ignore:
Timestamp:
Sep 20, 2012 11:41:46 AM (12 years ago)
Author:
dpranke@chromium.org
Message:

REGRESSION: layout test results doesn't show diffs
​https://bugs.webkit.org/show_bug.cgi?id=97182

Reviewed by Ojan Vafai.

Go back to storing TEXT, AUDIO, and IMAGE+TEXT in results.json
so that results.html (and hopefully garden-o-matic) can
determine which things actually failed. However, we keep mapping
these results to Failure so that we still only have a single
expectation type for them.

  • Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:

(JSONLayoutResultsGenerator):

  • Scripts/webkitpy/layout_tests/models/test_expectations.py:

(TestExpectationParser):
(TestExpectations):
(TestExpectations.result_was_expected):

  • Scripts/webkitpy/layout_tests/models/test_failures.py:

(determine_result_type):

  • Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

(MainTest.test_missing_and_unexpected_results):
(MainTest.test_retrying_and_flaky_tests):

Location:
trunk/Tools
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r129145 r129148  
     12012-09-20  Dirk Pranke  <dpranke@chromium.org>
     2
     3        REGRESSION: layout test results doesn't show diffs
     4        https://bugs.webkit.org/show_bug.cgi?id=97182
     5
     6        Reviewed by Ojan Vafai.
     7
     8        Go back to storing TEXT, AUDIO, and IMAGE+TEXT in results.json
     9        so that results.html (and hopefully garden-o-matic) can
     10        determine which things actually failed. However, we keep mapping
     11        these results to Failure so that we still only have a single
     12        expectation type for them.
     13
     14        * Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py:
     15        (JSONLayoutResultsGenerator):
     16        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
     17        (TestExpectationParser):
     18        (TestExpectations):
     19        (TestExpectations.result_was_expected):
     20        * Scripts/webkitpy/layout_tests/models/test_failures.py:
     21        (determine_result_type):
     22        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
     23        (MainTest.test_missing_and_unexpected_results):
     24        (MainTest.test_retrying_and_flaky_tests):
     25
    1262012-09-20  Tommy Widenflycht  <tommyw@google.com>
    227

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py

    r129047 r129148  
    4141    WONTFIX = "wontfixCounts"
    4242
    43     # Note that there used to be an "A" for audio failures and a "Z" for IMAGE+TEXT failures.
    44 
    4543    FAILURE_TO_CHAR = {test_expectations.PASS: json_results_generator.JSONResultsGeneratorBase.PASS_RESULT,
    4644                       test_expectations.SKIP: json_results_generator.JSONResultsGeneratorBase.SKIP_RESULT,
    4745                       test_expectations.CRASH: "C",
    4846                       test_expectations.TIMEOUT: "T",
    49                        test_expectations.FAIL: "F",
    5047                       test_expectations.IMAGE: "I",
    51                        test_expectations.MISSING: "O"}
     48                       test_expectations.TEXT: "F",
     49                       test_expectations.AUDIO: "A",
     50                       test_expectations.MISSING: "O",
     51                       test_expectations.IMAGE_PLUS_TEXT: "Z"}
    5252
    5353    def __init__(self, port, builder_name, build_name, build_number,

  • trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

    r129069 r129148  
    4141
    4242# Test expectation and modifier constants.
    43 # TEXT, IMAGE_PLUS_TEXT, and AUDIO are no longer used in new test runs but
    44 # we keep them around for now so we can parse old results.json entries and to
    45 # avoid changing the numbering for the constants.
    4643#
    4744# FIXME: range() starts with 0 which makes if expectation checks harder
    … …  
    287284    }
    288285
    289     _inverted_expectation_tokens = dict((value, name) for name, value in _expectation_tokens.iteritems())
     286    _inverted_expectation_tokens = dict([(value, name) for name, value in _expectation_tokens.iteritems()] +
     287                                        [('TEXT', 'Failure'), ('IMAGE+TEXT', 'Failure'), ('AUDIO', 'Failure')])
    290288
    291289    @classmethod
    … …  
    792790    # FIXME: Update to new syntax once the old format is no longer supported.
    793791    EXPECTATIONS = {'pass': PASS,
     792                    'audio': AUDIO,
    794793                    'fail': FAIL,
    795794                    'image': IMAGE,
     795                    'image+text': IMAGE_PLUS_TEXT,
     796                    'text': TEXT,
    796797                    'timeout': TIMEOUT,
    797798                    'crash': CRASH,
    … …  
    803804                                FAIL: ('failures', 'failed', ''),
    804805                                IMAGE: ('image-only failures', 'failed', ' (image diff)'),
     806                                TEXT: ('text-only failures', 'failed', ' (text diff)'),
     807                                IMAGE_PLUS_TEXT: ('image and text failures', 'failed', ' (image and text diff)'),
     808                                AUDIO: ('audio failures', 'failed', ' (audio diff)'),
    805809                                CRASH: ('crashes', 'crashed', ''),
    806810                                TIMEOUT: ('timeouts', 'timed out', ''),
    … …  
    839843            test_is_skipped: whether test was marked as SKIP"""
    840844        if result in expected_results:
     845            return True
     846        if result in (TEXT, IMAGE_PLUS_TEXT, AUDIO) and (FAIL in expected_results):
    841847            return True
    842848        if result == MISSING and test_needs_rebaselining:

  • trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py

    r129047 r129148  
    6363        return test_expectations.MISSING
    6464    else:
    65         if FailureTextMismatch in failure_types or FailureAudioMismatch in failure_types:
    66             return test_expectations.FAIL
    67         elif FailureImageHashIncorrect in failure_types or FailureImageHashMismatch in failure_types or is_reftest_failure(failure_list):
     65        is_text_failure = FailureTextMismatch in failure_types
     66        is_image_failure = (FailureImageHashIncorrect in failure_types or
     67                            FailureImageHashMismatch in failure_types)
     68        is_audio_failure = (FailureAudioMismatch in failure_types)
     69        if is_text_failure and is_image_failure:
     70            return test_expectations.IMAGE_PLUS_TEXT
     71        elif is_text_failure:
     72            return test_expectations.TEXT
     73        elif is_image_failure or is_reftest_failure(failure_list):
    6874            return test_expectations.IMAGE
     75        elif is_audio_failure:
     76            return test_expectations.AUDIO
    6977        else:
    7078            raise ValueError("unclassifiable set of failures: "

  • trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

    r129052 r129148  
    550550        file_list.remove('/tmp/layout-test-results/tests_run0.txt')
    551551        self.assertEquals(res, 1)
    552         expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"FAIL"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
     552        expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
    553553        json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
    554554        self.assertTrue(json_string.find(expected_token) != -1)
    … …  
    765765        self.assertTrue('Clobbering old results' in err.getvalue())
    766766        self.assertTrue('flaky/text.html' in err.getvalue())
    767         self.assertTrue('Unexpected failures' in out.getvalue())
     767        self.assertTrue('Unexpected text-only failures' in out.getvalue())
    768768        self.assertFalse('Unexpected flakiness' in out.getvalue())
    769769        self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
Note: See TracChangeset for help on using the changeset viewer.