Changeset 273145 in webkit


Ignore:
Timestamp:
Feb 19, 2021 10:21:18 AM (3 years ago)
Author:
aakash_jain@apple.com
Message:

[ews] Ensure that the test failures are listed in sorted order
​https://bugs.webkit.org/show_bug.cgi?id=222176

Reviewed by Jonathan Bedard.

  • CISupport/ews-build/steps.py:

(ReRunJavaScriptCoreTests.evaluateCommand):
(AnalyzeJSCTestsResults.start):
(RunWebKitTests.commandComplete):
(ReRunWebKitTests.evaluateCommand):
(ReRunWebKitTests.commandComplete):

  • CISupport/ews-build/steps_unittest.py:

(TestAnalyzeJSCTestsResults.test_multiple_new_stress_failure):
(TestAnalyzeJSCTestsResults.test_multiple_new_binary_failure):

Location:
trunk/Tools
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/CISupport/ews-build/steps.py

    r273090 r273145  
    17801780        second_run_failures = set(self.getProperty('jsc_rerun_stress_test_failures', []) + self.getProperty('jsc_rerun_binary_failures', []))
    17811781        flaky_failures = first_run_failures.union(second_run_failures) - first_run_failures.intersection(second_run_failures)
    1782         flaky_failures_string = ', '.join(flaky_failures)
     1782        flaky_failures_string = ', '.join(sorted(flaky_failures))
    17831783
    17841784        if rc == SUCCESS or rc == WARNINGS:
    … …  
    18291829        flaky_stress_failures = first_run_stress_failures.union(second_run_stress_failures) - first_run_stress_failures.intersection(second_run_stress_failures)
    18301830        flaky_binary_failures = first_run_binary_failures.union(second_run_binary_failures) - first_run_binary_failures.intersection(second_run_binary_failures)
    1831         flaky_failures = (list(flaky_binary_failures) + list(flaky_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]
     1831        flaky_failures = sorted(list(flaky_binary_failures) + list(flaky_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]
    18321832        flaky_failures_string = ', '.join(flaky_failures)
    18331833
    18341834        new_stress_failures = stress_failures_with_patch - clean_tree_stress_failures
    18351835        new_binary_failures = binary_failures_with_patch - clean_tree_binary_failures
    1836         new_stress_failures_to_display = ', '.join(list(new_stress_failures)[:self.NUM_FAILURES_TO_DISPLAY])
    1837         new_binary_failures_to_display = ', '.join(list(new_binary_failures)[:self.NUM_FAILURES_TO_DISPLAY])
     1836        new_stress_failures_to_display = ', '.join(sorted(list(new_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY])
     1837        new_binary_failures_to_display = ', '.join(sorted(list(new_binary_failures))[:self.NUM_FAILURES_TO_DISPLAY])
    18381838
    18391839        self._addToLog('stderr', '\nFailures in first run: {}'.format((list(first_run_binary_failures) + list(first_run_stress_failures))[:self.NUM_FAILURES_TO_DISPLAY]))
    … …  
    20762076        if first_results:
    20772077            self.setProperty('first_results_exceed_failure_limit', first_results.did_exceed_test_failure_limit)
    2078             self.setProperty('first_run_failures', first_results.failing_tests)
     2078            self.setProperty('first_run_failures', sorted(first_results.failing_tests))
    20792079            if first_results.failing_tests:
    20802080                self._addToLog(self.test_failures_log_name, '\n'.join(first_results.failing_tests))
    … …  
    21432143        tests_that_consistently_failed = first_results_failing_tests.intersection(second_results_failing_tests)
    21442144        flaky_failures = first_results_failing_tests.union(second_results_failing_tests) - first_results_failing_tests.intersection(second_results_failing_tests)
    2145         flaky_failures = list(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY]
     2145        flaky_failures = sorted(list(flaky_failures))[:self.NUM_FAILURES_TO_DISPLAY]
    21462146        flaky_failures_string = ', '.join(flaky_failures)
    21472147
    … …  
    21782178        if second_results:
    21792179            self.setProperty('second_results_exceed_failure_limit', second_results.did_exceed_test_failure_limit)
    2180             self.setProperty('second_run_failures', second_results.failing_tests)
     2180            self.setProperty('second_run_failures', sorted(second_results.failing_tests))
    21812181            if second_results.failing_tests:
    21822182                self._addToLog(self.test_failures_log_name, '\n'.join(second_results.failing_tests))

  • trunk/Tools/CISupport/ews-build/steps_unittest.py

    r273090 r273145  
    15071507        self.setProperty('jsc_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
    15081508        self.setProperty('jsc_rerun_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
    1509         self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test1, test0, test3, test2, test5, test4, test7, test6, test9, test8 ... (failure)')
     1509        self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test0, test1, test10, test11, test12, test13, test14, test15, test16, test17 ... (failure)')
    15101510        return self.runStep()
    15111511
    … …  
    15141514        self.setProperty('jsc_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
    15151515        self.setProperty('jsc_rerun_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
    1516         self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testb3, testmasm, testapi, testdfg, testair (failure)')
     1516        self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testair, testapi, testb3, testdfg, testmasm (failure)')
    15171517        return self.runStep()
    15181518
    … …  
    16741674        self.assertEqual(self.getProperty(self.property_exceed_failure_limit), True)
    16751675        self.assertEqual(self.getProperty(self.property_failures),
    1676                             ["imported/w3c/web-platform-tests/IndexedDB/interleaved-cursors-large.html",
    1677                              "imported/w3c/web-platform-tests/wasm/jsapi/interface.any.html",
    1678                              "imported/w3c/web-platform-tests/wasm/jsapi/instance/constructor-bad-imports.any.html",
    1679                              "imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.html",
    1680                              "imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.worker.html",
    1681                              "imported/w3c/web-platform-tests/wasm/jsapi/global/toString.any.html",
    1682                              "imported/w3c/web-platform-tests/wasm/jsapi/interface.any.worker.html",
    1683                              "imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.html",
    1684                              "imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.worker.html",
    1685                              "imported/blink/storage/indexeddb/blob-valid-before-commit.html"])
     1676                            ['imported/blink/storage/indexeddb/blob-valid-before-commit.html',
     1677                             'imported/w3c/web-platform-tests/IndexedDB/interleaved-cursors-large.html',
     1678                             'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.html',
     1679                             'imported/w3c/web-platform-tests/wasm/jsapi/constructor/instantiate-bad-imports.any.worker.html',
     1680                             'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.html',
     1681                             'imported/w3c/web-platform-tests/wasm/jsapi/global/constructor.any.worker.html',
     1682                             'imported/w3c/web-platform-tests/wasm/jsapi/global/toString.any.html',
     1683                             'imported/w3c/web-platform-tests/wasm/jsapi/instance/constructor-bad-imports.any.html',
     1684                             'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.html',
     1685                             'imported/w3c/web-platform-tests/wasm/jsapi/interface.any.worker.html'])
    16861686        return rc
    16871687

  • trunk/Tools/ChangeLog

    r273142 r273145  
     12021-02-19  Aakash Jain  <aakash_jain@apple.com>
     2
     3        [ews] Ensure that the test failures are listed in sorted order
     4        https://bugs.webkit.org/show_bug.cgi?id=222176
     5
     6        Reviewed by Jonathan Bedard.
     7
     8        * CISupport/ews-build/steps.py:
     9        (ReRunJavaScriptCoreTests.evaluateCommand):
     10        (AnalyzeJSCTestsResults.start):
     11        (RunWebKitTests.commandComplete):
     12        (ReRunWebKitTests.evaluateCommand):
     13        (ReRunWebKitTests.commandComplete):
     14        * CISupport/ews-build/steps_unittest.py:
     15        (TestAnalyzeJSCTestsResults.test_multiple_new_stress_failure):
     16        (TestAnalyzeJSCTestsResults.test_multiple_new_binary_failure):
     17
    1182021-02-19  Jonathan Bedard  <jbedard@apple.com>
    219
Note: See TracChangeset for help on using the changeset viewer.