Changeset 252446 in webkit

Timestamp:

Nov 13, 2019 7:07:35 PM (4 years ago)

Author:

aakash_jain@apple.com

Message:

[ews] Add build step to Analyze JSC Tests Results
​https://bugs.webkit.org/show_bug.cgi?id=204174

Reviewed by Jonathan Bedard.

• BuildSlaveSupport/ews-build/steps.py:

(ReRunJavaScriptCoreTests.evaluateCommand): invoke AnalyzeJSCTestsResults step.
(AnalyzeJSCTestsResults): Build step to analyze layout-test results.

• BuildSlaveSupport/ews-build/steps_unittest.py:

(TestAnalyzeJSCTestsResults):
(TestAnalyzeJSCTestsResults.test_single_new_stress_failure): Added unit-test.
(TestAnalyzeJSCTestsResults.test_single_new_binary_failure): Ditto.
(TestAnalyzeJSCTestsResults.test_multiple_new_stress_failure): Ditto.
(TestAnalyzeJSCTestsResults.test_multiple_new_binary_failure): Ditto.
(TestAnalyzeJSCTestsResults.test_new_stress_and_binary_failure): Ditto.
(TestAnalyzeJSCTestsResults.test_stress_failure_on_clean_tree): Ditto.
(TestAnalyzeJSCTestsResults.test_binary_failure_on_clean_tree): Ditto.
(TestAnalyzeJSCTestsResults.test_stress_and_binary_failure_on_clean_tree): Ditto.
(TestAnalyzeJSCTestsResults.test_flaky_stress_and_binary_failures): Ditto.
(TestAnalyzeJSCTestsResults.test_flaky_and_consistent_stress_failures): Ditto.
(TestAnalyzeJSCTestsResults.test_flaky_and_consistent_failures_with_clean_tree_failures): Ditto.

Location:

trunk/Tools

Files:

• BuildSlaveSupport/ews-build/steps.py (modified) (2 diffs)
• BuildSlaveSupport/ews-build/steps_unittest.py (modified) (2 diffs)
• ChangeLog (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/ews-build/steps.py

@@ -1047 +1047 @@
         else:
             self.setProperty('patchFailedTests', True)
-            self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileJSCToT(), RunJSCTestsWithoutPatch()])
+            self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileJSCToT(), RunJSCTestsWithoutPatch(), AnalyzeJSCTestsResults()])
         return rc
 
@@ -1057 +1057 @@
     def evaluateCommand(self, cmd):
         return shell.Test.evaluateCommand(self, cmd)
+
+
+class AnalyzeJSCTestsResults(buildstep.BuildStep):
+    name = 'analyze-jsc-tests-results'
+    description = ['analyze-jsc-test-results']
+    descriptionDone = ['analyze-jsc-tests-results']
+    NUM_FAILURES_TO_DISPLAY = 10
+
+    def start(self):
+        first_run_stress_failures = set(self.getProperty('jsc_stress_test_failures', []))
+        first_run_binary_failures = set(self.getProperty('jsc_binary_failures', []))
+        second_run_stress_failures = set(self.getProperty('jsc_rerun_stress_test_failures', []))
+        second_run_binary_failures = set(self.getProperty('jsc_rerun_binary_failures', []))
+        clean_tree_stress_failures = set(self.getProperty('jsc_clean_tree_stress_test_failures', []))
+        clean_tree_binary_failures = set(self.getProperty('jsc_clean_tree_binary_failures', []))
+        clean_tree_failures = list(clean_tree_binary_failures) + list(clean_tree_stress_failures)
+        clean_tree_failures_string = ', '.join(clean_tree_failures)
+
+        stress_failures_with_patch = first_run_stress_failures.intersection(second_run_stress_failures)
+        binary_failures_with_patch = first_run_binary_failures.intersection(second_run_binary_failures)
+
+        flaky_stress_failures = first_run_stress_failures.union(second_run_stress_failures) - first_run_stress_failures.intersection(second_run_stress_failures)
+        flaky_binary_failures = first_run_binary_failures.union(second_run_binary_failures) - first_run_binary_failures.intersection(second_run_binary_failures)
+        flaky_failures = list(flaky_binary_failures) + list(flaky_stress_failures)
+        flaky_failures_string = ', '.join(flaky_failures)
+
+        new_stress_failures = stress_failures_with_patch - clean_tree_stress_failures
+        new_binary_failures = binary_failures_with_patch - clean_tree_binary_failures
+        new_stress_failures_to_display = ', '.join(list(new_stress_failures)[:self.NUM_FAILURES_TO_DISPLAY])
+        new_binary_failures_to_display = ', '.join(list(new_binary_failures)[:self.NUM_FAILURES_TO_DISPLAY])
+
+        self._addToLog('stderr', '\nFailures in first run: {}'.format(list(first_run_binary_failures) + list(first_run_stress_failures)))
+        self._addToLog('stderr', '\nFailures in second run: {}'.format(list(second_run_binary_failures) + list(second_run_stress_failures)))
+        self._addToLog('stderr', '\nFlaky Tests: {}'.format(flaky_failures_string))
+        self._addToLog('stderr', '\nFailures on clean tree: {}'.format(list(clean_tree_stress_failures) + list(clean_tree_binary_failures)))
+
+        if new_stress_failures or new_binary_failures:
+            self._addToLog('stderr', '\nNew failures: {}\n'.format(list(new_binary_failures) + list(new_stress_failures)))
+            self.finished(FAILURE)
+            self.build.results = FAILURE
+            message = ''
+            if new_binary_failures:
+                pluralSuffix = 's' if len(new_binary_failures) > 1 else ''
+                message = 'Found {} new JSC binary failure{}: {}'.format(len(new_binary_failures), pluralSuffix, new_binary_failures_to_display)
+            if new_stress_failures:
+                if message:
+                    message += ', '
+                pluralSuffix = 's' if len(new_stress_failures) > 1 else ''
+                message += 'Found {} new JSC stress test failure{}: {}'.format(len(new_stress_failures), pluralSuffix, new_stress_failures_to_display)
+                if len(new_stress_failures) > self.NUM_FAILURES_TO_DISPLAY:
+                    message += ' ...'
+            self.descriptionDone = message
+            self.build.buildFinished([message], FAILURE)
+        else:
+            self._addToLog('stderr', '\nNo new failures\n')
+            self.finished(SUCCESS)
+            self.build.results = SUCCESS
+            self.descriptionDone = 'Passed JSC tests'
+            pluralSuffix = 's' if len(clean_tree_failures) > 1 else ''
+            message = ''
+            if clean_tree_failures:
+                message = 'Found {} pre-existing JSC test failure{}: {}'.format(len(clean_tree_failures), pluralSuffix, clean_tree_failures_string)
+            if len(clean_tree_failures) > self.NUM_FAILURES_TO_DISPLAY:
+                message += ' ...'
+            if flaky_failures:
+                message += ' Found flaky tests: {}'.format(flaky_failures_string)
+            self.build.buildFinished([message], SUCCESS)
+        return defer.succeed(None)
+
+    @defer.inlineCallbacks
+    def _addToLog(self, logName, message):
+        try:
+            log = self.getLog(logName)
+        except KeyError:
+            log = yield self.addLog(logName)
+        log.addStdout(message)
 
 

trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py

@@ -35 +35 @@
 from twisted.trial import unittest
 
-from steps import (AnalyzeAPITestsResults, AnalyzeCompileWebKitResults, AnalyzeLayoutTestsResults, ApplyPatch, ApplyWatchList, ArchiveBuiltProduct, ArchiveTestResults,
+from steps import (AnalyzeAPITestsResults, AnalyzeCompileWebKitResults, AnalyzeJSCTestsResults, AnalyzeLayoutTestsResults, ApplyPatch, ApplyWatchList, ArchiveBuiltProduct, ArchiveTestResults,
                    CheckOutSource, CheckOutSpecificRevision, CheckPatchRelevance, CheckStyle, CleanBuild, CleanUpGitIndexLock, CleanWorkingDirectory,
                    CompileJSC, CompileJSCToT, CompileWebKit, CompileWebKitToT, ConfigureBuild,
@@ -1197 +1197 @@
         )
         self.expectOutcome(result=FAILURE, state_string='jscore-tests (failure)')
+        return self.runStep()
+
+
+class TestAnalyzeJSCTestsResults(BuildStepMixinAdditions, unittest.TestCase):
+    def setUp(self):
+        self.longMessage = True
+        return self.setUpBuildStep()
+
+    def tearDown(self):
+        return self.tearDownBuildStep()
+
+    def configureStep(self):
+        self.setupStep(AnalyzeJSCTestsResults())
+        self.setProperty('jsc_stress_test_failures', [])
+        self.setProperty('jsc_binary_failures', [])
+        self.setProperty('jsc_rerun_stress_test_failures', [])
+        self.setProperty('jsc_rerun_binary_failures', [])
+        self.setProperty('jsc_clean_tree_stress_test_failures', [])
+        self.setProperty('jsc_clean_tree_binary_failures', [])
+
+    def test_single_new_stress_failure(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.bytecode-cache'])
+        self.setProperty('jsc_rerun_stress_test_failures', ['stress/force-error.js.bytecode-cache'])
+        self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC stress test failure: stress/force-error.js.bytecode-cache (failure)')
+        return self.runStep()
+
+    def test_single_new_binary_failure(self):
+        self.configureStep()
+        self.setProperty('jsc_binary_failures', ['testmasm'])
+        self.setProperty('jsc_rerun_binary_failures', ['testmasm'])
+        self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC binary failure: testmasm (failure)')
+        return self.runStep()
+
+    def test_multiple_new_stress_failure(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
+        self.setProperty('jsc_rerun_stress_test_failures', ['test{}'.format(i) for i in range(0, 30)])
+        self.expectOutcome(result=FAILURE, state_string='Found 30 new JSC stress test failures: test1, test0, test3, test2, test5, test4, test7, test6, test9, test8 ... (failure)')
+        return self.runStep()
+
+    def test_multiple_new_binary_failure(self):
+        self.configureStep()
+        self.setProperty('jsc_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
+        self.setProperty('jsc_rerun_binary_failures', ['testmasm', 'testair', 'testb3', 'testdfg', 'testapi'])
+        self.expectOutcome(result=FAILURE, state_string='Found 5 new JSC binary failures: testb3, testmasm, testapi, testdfg, testair (failure)')
+        return self.runStep()
+
+    def test_new_stress_and_binary_failure(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+        self.setProperty('jsc_binary_failures', ['testmasm'])
+        self.setProperty('jsc_rerun_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+        self.setProperty('jsc_rerun_binary_failures', ['testmasm'])
+        self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC binary failure: testmasm, Found 1 new JSC stress test failure: es6.yaml/es6/Set_iterator_closing.js.default (failure)')
+        return self.runStep()
+
+    def test_stress_failure_on_clean_tree(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.default'])
+        self.setProperty('jsc_rerun_stress_test_failures', ['stress/force-error.js.default'])
+        self.setProperty('jsc_clean_tree_stress_test_failures', ['stress/force-error.js.default'])
+        self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+        return self.runStep()
+
+    def test_binary_failure_on_clean_tree(self):
+        self.configureStep()
+        self.setProperty('jsc_binary_failures', ['testdfg'])
+        self.setProperty('jsc_rerun_binary_failures', ['testdfg'])
+        self.setProperty('jsc_clean_tree_binary_failures', ['testdfg'])
+        self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+        return self.runStep()
+
+    def test_stress_and_binary_failure_on_clean_tree(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+        self.setProperty('jsc_binary_failures', ['testair'])
+        self.setProperty('jsc_rerun_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+        self.setProperty('jsc_rerun_binary_failures', ['testair'])
+        self.setProperty('jsc_clean_tree_stress_test_failures', ['es6.yaml/es6/Set_iterator_closing.js.default'])
+        self.setProperty('jsc_clean_tree_binary_failures', ['testair'])
+        self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+        return self.runStep()
+
+    def test_flaky_stress_and_binary_failures(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['stress/force-error.js.default'])
+        self.setProperty('jsc_binary_failures', ['testapi'])
+        self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
+        return self.runStep()
+
+    def test_flaky_and_consistent_stress_failures(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['test1', 'test2'])
+        self.setProperty('jsc_rerun_stress_test_failures', ['test2'])
+        self.expectOutcome(result=FAILURE, state_string='Found 1 new JSC stress test failure: test2 (failure)')
+        return self.runStep()
+
+    def test_flaky_and_consistent_failures_with_clean_tree_failures(self):
+        self.configureStep()
+        self.setProperty('jsc_stress_test_failures', ['test1', 'test2'])
+        self.setProperty('jsc_rerun_stress_test_failures', ['test1'])
+        self.setProperty('jsc_clean_tree_stress_test_failures', ['test1', 'test2'])
+        self.expectOutcome(result=SUCCESS, state_string='Passed JSC tests')
         return self.runStep()
 

trunk/Tools/ChangeLog

@@ +1 @@
+2019-11-13  Aakash Jain  <aakash_jain@apple.com>
+
+        [ews] Add build step to Analyze JSC Tests Results
+        https://bugs.webkit.org/show_bug.cgi?id=204174
+
+        Reviewed by Jonathan Bedard.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (ReRunJavaScriptCoreTests.evaluateCommand): invoke AnalyzeJSCTestsResults step.
+        (AnalyzeJSCTestsResults): Build step to analyze layout-test results.
+        * BuildSlaveSupport/ews-build/steps_unittest.py:
+        (TestAnalyzeJSCTestsResults):
+        (TestAnalyzeJSCTestsResults.test_single_new_stress_failure): Added unit-test.
+        (TestAnalyzeJSCTestsResults.test_single_new_binary_failure): Ditto.
+        (TestAnalyzeJSCTestsResults.test_multiple_new_stress_failure): Ditto.
+        (TestAnalyzeJSCTestsResults.test_multiple_new_binary_failure): Ditto.
+        (TestAnalyzeJSCTestsResults.test_new_stress_and_binary_failure): Ditto.
+        (TestAnalyzeJSCTestsResults.test_stress_failure_on_clean_tree): Ditto.
+        (TestAnalyzeJSCTestsResults.test_binary_failure_on_clean_tree): Ditto.
+        (TestAnalyzeJSCTestsResults.test_stress_and_binary_failure_on_clean_tree): Ditto.
+        (TestAnalyzeJSCTestsResults.test_flaky_stress_and_binary_failures): Ditto.
+        (TestAnalyzeJSCTestsResults.test_flaky_and_consistent_stress_failures): Ditto.
+        (TestAnalyzeJSCTestsResults.test_flaky_and_consistent_failures_with_clean_tree_failures): Ditto.
+
 2019-11-13  Jonathan Bedard  <jbedard@apple.com>