Changeset 243342 in webkit

Timestamp:

Mar 21, 2019, 4:11:58 PM (7 years ago)

Author:

aakash_jain@apple.com

Message:

[ews-build] Retry API test in case of failures
​https://bugs.webkit.org/show_bug.cgi?id=196004

Reviewed by Lucas Forschler.

• BuildSlaveSupport/ews-build/steps.py:

(UnApplyPatchIfRequired.doStepIf): Updated doStepIf to include patchFailedAPITests.
(CompileWebKitToT.doStepIf): Ditto.
(RunAPITests.evaluateCommand): Check if tests failed and retry them if required.
(ReRunAPITests): Re-run API tests.
(ReRunAPITests.evaluateCommand): Check if tests failed and retry on clean build if required.
(RunAPITestsWithoutPatch): Run API tests without patch.
(RunAPITestsWithoutPatch.doStepIf):
(RunAPITestsWithoutPatch.hideStepIf):
(RunAPITestsWithoutPatch.evaluateCommand):
(AnalyzeAPITestsResults): Analyze API test results from previous runs.
(AnalyzeAPITestsResults.start):
(AnalyzeAPITestsResults.analyzeResults): Analyze API test results.
(AnalyzeAPITestsResults.getBuildStepByName): Search for a build step by name.
(AnalyzeAPITestsResults.getTestsResults): Get the test results from previous API tesst steps.

• BuildSlaveSupport/ews-build/steps_unittest.py: Monkey patched FakeBuild.

Location:

trunk/Tools

Files:

• BuildSlaveSupport/ews-build/steps.py (modified) (4 diffs)
• BuildSlaveSupport/ews-build/steps_unittest.py (modified) (1 diff)
• ChangeLog (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/ews-build/steps.py

@@ -29 +29 @@
 from twisted.internet import defer
 
+import json
 import re
 import requests
@@ -377 +378 @@
 
     def doStepIf(self, step):
-        return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedJSCTests')
+        return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedJSCTests') or self.getProperty('patchFailedAPITests')
 
     def hideStepIf(self, results, step):
@@ -527 +528 @@
 
     def doStepIf(self, step):
-        return self.getProperty('patchFailedToBuild')
+        return self.getProperty('patchFailedToBuild') or self.getProperty('patchFailedAPITests')
 
     def hideStepIf(self, results, step):
@@ -711 +712 @@
         return int(match.group('ran')) - int(match.group('passed'))
 
+    def evaluateCommand(self, cmd):
+        rc = super(RunAPITests, self).evaluateCommand(cmd)
+        if rc == SUCCESS:
+            message = 'Passed API tests'
+            self.descriptionDone = message
+            self.build.results = SUCCESS
+            self.build.buildFinished([message], SUCCESS)
+        else:
+            self.build.addStepsAfterCurrentStep([ReRunAPITests()])
+        return rc
+
+
+class ReRunAPITests(RunAPITests):
+    name = 're-run-api-tests'
+
+    def evaluateCommand(self, cmd):
+        rc = TestWithFailureCount.evaluateCommand(self, cmd)
+        if rc == SUCCESS:
+            message = 'Passed API tests'
+            self.descriptionDone = message
+            self.build.results = SUCCESS
+            self.build.buildFinished([message], SUCCESS)
+        else:
+            self.setProperty('patchFailedAPITests', True)
+            self.build.addStepsAfterCurrentStep([UnApplyPatchIfRequired(), CompileWebKitToT(), RunAPITestsWithoutPatch(), AnalyzeAPITestsResults()])
+        return rc
+
+
+class RunAPITestsWithoutPatch(RunAPITests):
+    name = 'run-api-tests-without-patch'
+
+    def evaluateCommand(self, cmd):
+        return TestWithFailureCount.evaluateCommand(self, cmd)
+
+
+class AnalyzeAPITestsResults(buildstep.BuildStep):
+    name = 'analyze-api-tests-results'
+    description = ['analyze-api-test-results']
+    descriptionDone = ['analyze-api-tests-results']
+
+    def start(self):
+        self.results = {}
+        d = self.getTestsResults(RunAPITests.name)
+        d.addCallback(lambda res: self.getTestsResults(ReRunAPITests.name))
+        d.addCallback(lambda res: self.getTestsResults(RunAPITestsWithoutPatch.name))
+        d.addCallback(lambda res: self.analyzeResults())
+        return defer.succeed(None)
+
+    def analyzeResults(self):
+        if not self.results or len(self.results) == 0:
+            self._addToLog('stderr', 'Unable to parse API test results: {}'.format(self.results))
+            self.finished(RETRY)
+            self.build.buildFinished(['Unable to parse API test results'], RETRY)
+            return -1
+
+        first_run_results = self.results.get(RunAPITests.name)
+        second_run_results = self.results.get(ReRunAPITests.name)
+        clean_tree_results = self.results.get(RunAPITestsWithoutPatch.name)
+
+        if not (first_run_results and second_run_results and clean_tree_results):
+            self.finished(RETRY)
+            self.build.buildFinished(['Unable to parse API test results'], RETRY)
+            return -1
+
+        def getAPITestFailures(result):
+            # TODO: Analyze Time-out, Crash and Failure independently
+            return set([failure.get('name') for failure in result.get('Timedout', [])] +
+                [failure.get('name') for failure in result.get('Crashed', [])] +
+                [failure.get('name') for failure in result.get('Failed', [])])
+
+        first_run_failures = getAPITestFailures(first_run_results)
+        second_run_failures = getAPITestFailures(second_run_results)
+        clean_tree_failures = getAPITestFailures(clean_tree_results)
+
+        self._addToLog('stderr', '\nFailures in API Test first run: {}'.format(first_run_failures))
+        self._addToLog('stderr', '\nFailures in API Test second run: {}'.format(first_run_failures))
+        self._addToLog('stderr', '\nFailures in API Test on clean tree: {}'.format(clean_tree_failures))
+        failures_with_patch = first_run_failures.intersection(second_run_failures)
+        new_failures = failures_with_patch - clean_tree_failures
+        new_failures_string = ', '.join([failure_name.replace('TestWebKitAPI.', '') for failure_name in new_failures])
+
+        if new_failures:
+            self._addToLog('stderr', '\nNew failures: {}\n'.format(new_failures))
+            self.finished(FAILURE)
+            self.build.results = FAILURE
+            message = 'Found {} new API Tests failures: {}'.format(len(new_failures), new_failures_string)
+            self.descriptionDone = message
+            self.build.buildFinished([message], FAILURE)
+        else:
+            self._addToLog('stderr', '\nNo new failures\n')
+            self.finished(SUCCESS)
+            self.build.results = SUCCESS
+            self.descriptionDone = 'Passed API tests'
+            message = 'Found {} pre-existing API tests failures'.format(len(clean_tree_failures))
+            self.build.buildFinished([message], SUCCESS)
+
+    @defer.inlineCallbacks
+    def _addToLog(self, logName, message):
+        try:
+            log = self.getLog(logName)
+        except KeyError:
+            log = yield self.addLog(logName)
+        log.addStdout(message)
+
+    def getBuildStepByName(self, name):
+        for step in self.build.executedSteps:
+            if step.name == name:
+                return step
+        return None
+
+    @defer.inlineCallbacks
+    def getTestsResults(self, name):
+        step = self.getBuildStepByName(name)
+        if not step:
+            self._addToLog('stderr', 'ERROR: step not found: {}'.format(step))
+            defer.returnValue(None)
+
+        logs = yield self.master.db.logs.getLogs(step.stepid)
+        log = next((log for log in logs if log['name'] == u'json'), None)
+        if not log:
+            self._addToLog('stderr', 'ERROR: log for step not found: {}'.format(step))
+            defer.returnValue(None)
+
+        lastline = int(max(0, log['num_lines'] - 1))
+        logLines = yield self.master.db.logs.getLogLines(log['id'], 0, lastline)
+        if log['type'] == 's':
+            logLines = ''.join([line[1:] for line in logLines.splitlines()])
+
+        try:
+            self.results[name] = json.loads(logLines)
+        except Exception as ex:
+            self._addToLog('stderr', 'ERROR: unable to parse data, exception: {}'.format(ex))
+
 
 class ArchiveTestResults(shell.ShellCommand):

trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py

@@ -37 +37 @@
 from steps import *
 
+# Workaround for https://github.com/buildbot/buildbot/issues/4669
+from buildbot.test.fake.fakebuild import FakeBuild
+FakeBuild.addStepsAfterCurrentStep = lambda FakeBuild, step_factories: None
+
 
 class ExpectMasterShellCommand(object):

trunk/Tools/ChangeLog

@@ +1 @@
+2019-03-21  Aakash Jain  <aakash_jain@apple.com>
+
+        [ews-build] Retry API test in case of failures
+        https://bugs.webkit.org/show_bug.cgi?id=196004
+
+        Reviewed by Lucas Forschler.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (UnApplyPatchIfRequired.doStepIf): Updated doStepIf to include patchFailedAPITests.
+        (CompileWebKitToT.doStepIf): Ditto.
+        (RunAPITests.evaluateCommand): Check if tests failed and retry them if required.
+        (ReRunAPITests): Re-run API tests.
+        (ReRunAPITests.evaluateCommand): Check if tests failed and retry on clean build if required.
+        (RunAPITestsWithoutPatch): Run API tests without patch.
+        (RunAPITestsWithoutPatch.doStepIf):
+        (RunAPITestsWithoutPatch.hideStepIf):
+        (RunAPITestsWithoutPatch.evaluateCommand):
+        (AnalyzeAPITestsResults): Analyze API test results from previous runs.
+        (AnalyzeAPITestsResults.start):
+        (AnalyzeAPITestsResults.analyzeResults): Analyze API test results.
+        (AnalyzeAPITestsResults.getBuildStepByName): Search for a build step by name.
+        (AnalyzeAPITestsResults.getTestsResults): Get the test results from previous API tesst steps.
+        * BuildSlaveSupport/ews-build/steps_unittest.py: Monkey patched FakeBuild.
+
 2019-03-21  Aakash Jain  <aakash_jain@apple.com>