Changeset 161171 in webkit


Ignore:
Timestamp:
Dec 30, 2013, 10:32:59 PM (11 years ago)
Author:
ap@apple.com
Message:

full_results.json should distinguish unexpected failures from expected ones
​https://bugs.webkit.org/show_bug.cgi?id=126300

Reviewed by Timothy Hatcher.

  • Scripts/webkitpy/layout_tests/models/test_run_results.py:

(summarize_results): Add "report" element to JSON, which tells the consumer how
this result was counted for summary.

  • Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

Updated results to include the new element.

  • Scripts/webkitpy/layout_tests/views/buildbot_results.py:

(print_unexpected_results): Added a comment pointing to another place that
summarizes results, and should stay in sync.

Location:
trunk/Tools
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r161156 r161171  
     12013-12-30  Alexey Proskuryakov  <ap@apple.com>
     2
     3        full_results.json should distinguish unexpected failures from expected ones
     4        https://bugs.webkit.org/show_bug.cgi?id=126300
     5
     6        Reviewed by Timothy Hatcher.
     7
     8        * Scripts/webkitpy/layout_tests/models/test_run_results.py:
     9        (summarize_results): Add "report" element to JSON, which tells the consumer how
     10        this result was counted for summary.
     11
     12        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
     13        Updated results to include the new element.
     14
     15        * Scripts/webkitpy/layout_tests/views/buildbot_results.py:
     16        (print_unexpected_results): Added a comment pointing to another place that
     17        summarizes results, and should stay in sync.
     18
    1192013-12-30  Ryuan Choi  <ryuan.choi@samsung.com>
    220

  • trunk/Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py

    r160271 r161171  
    118118    return test_dict
    119119
    120 
     120# These results must match ones in print_unexpected_results() in views/buildbot_results.py.
    121121def summarize_results(port_obj, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, include_passes=False, include_time_and_modifiers=False):
    122122    """Returns a dictionary containing a summary of the test runs, with the following fields:
    … …  
    180180            if test_name in initial_results.unexpected_results_by_name:
    181181                num_regressions += 1
     182                test_dict['report'] = 'REGRESSION'
    182183        elif result_type == test_expectations.MISSING:
    183184            if test_name in initial_results.unexpected_results_by_name:
    184185                num_missing += 1
     186                test_dict['report'] = 'MISSING'
    185187        elif test_name in initial_results.unexpected_results_by_name:
    186188            if retry_results and test_name not in retry_results.unexpected_results_by_name:
    187189                actual.extend(expectations.model().get_expectations_string(test_name).split(" "))
    188190                num_flaky += 1
     191                test_dict['report'] = 'FLAKY'
    189192            elif retry_results:
    190193                retry_result_type = retry_results.unexpected_results_by_name[test_name].type
    … …  
    192195                    if enabled_pixel_tests_in_retry and result_type == test_expectations.TEXT and retry_result_type == test_expectations.IMAGE_PLUS_TEXT:
    193196                        num_regressions += 1
     197                        test_dict['report'] = 'REGRESSION'
    194198                    else:
    195199                        num_flaky += 1
     200                        test_dict['report'] = 'FLAKY'
    196201                    actual.append(keywords[retry_result_type])
    197202                else:
    198203                    num_regressions += 1
     204                    test_dict['report'] = 'REGRESSION'
    199205            else:
    200206                num_regressions += 1
     207                test_dict['report'] = 'REGRESSION'
    201208
    202209        test_dict['expected'] = expected

  • trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

    r159675 r161171  
    497497        file_list = host.filesystem.written_files.keys()
    498498        self.assertEqual(details.exit_code, 1)
    499         expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"IMAGE+TEXT","image_diff_percent":1},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
     499        expected_token = '"unexpected":{"text-image-checksum.html":{"report":"REGRESSION","expected":"PASS","actual":"IMAGE+TEXT","image_diff_percent":1},"missing_text.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
    500500        json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
    501501        self.assertTrue(json_string.find(expected_token) != -1)
    … …  
    514514
    515515        self.assertEqual(details.exit_code, 1)
    516         expected_token = '"unexpected":{"pixeldir":{"image_in_pixeldir.html":{"expected":"PASS","actual":"IMAGE"'
     516        expected_token = '"unexpected":{"pixeldir":{"image_in_pixeldir.html":{"report":"REGRESSION","expected":"PASS","actual":"IMAGE"'
    517517        json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
    518518        self.assertTrue(json_string.find(expected_token) != -1)
    … …  
    538538        host = MockHost()
    539539        _, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html'], tests_included=True, host=host)
    540         self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
     540        self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"report":"REGRESSION","expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
    541541
    542542    def test_no_image_failure_with_image_diff(self):
    … …  
    668668        json = parse_full_results(json_string)
    669669        self.assertEqual(json["tests"]["failures"]["unexpected"]["text-image-checksum.html"],
    670             {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1})
     670            {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1, "report": "REGRESSION"})
    671671        self.assertFalse(json["pixel_tests_enabled"])
    672672        self.assertEqual(details.enabled_pixel_tests_in_retry, True)
    … …  
    750750        _, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
    751751        json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
    752         self.assertTrue(json_string.find('"unlistedtest.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
     752        self.assertTrue(json_string.find('"unlistedtest.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
    753753        self.assertTrue(json_string.find('"num_regressions":4') != -1)
    754754        self.assertTrue(json_string.find('"num_flaky":0') != -1)
    … …  
    851851        self.assertTrue("multiple-both-success.html" not in json["tests"]["reftests"]["foo"])
    852852        self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-match-failure.html"],
    853             {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "image_diff_percent": 1})
     853            {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "image_diff_percent": 1, "report": "REGRESSION"})
    854854        self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-mismatch-failure.html"],
    855             {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="]})
     855            {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="], "report": "REGRESSION"})
    856856        self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-both-failure.html"],
    857             {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["==", "!="]})
     857            {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["==", "!="], "report": "REGRESSION"})
    858858
    859859

  • trunk/Tools/Scripts/webkitpy/layout_tests/views/buildbot_results.py

    r146817 r161171  
    8989                self._print("  %5d %-24s (%4.1f%%)" % (len(results), desc, pct))
    9090
     91    # These results must match ones in summarize_results() in models/test_run_results.py.
    9192    def print_unexpected_results(self, summarized_results, enabled_pixel_tests_in_retry=False):
    9293        passes = {}
Note: See TracChangeset for help on using the changeset viewer.