Changeset 247433 in webkit

Timestamp:

Jul 15, 2019 9:54:28 AM (5 years ago)

Author:

aakash_jain@apple.com

Message:

[ews-build] Parse and display layout test failures
​https://bugs.webkit.org/show_bug.cgi?id=199709

Rubber-stamped by Jonathan Bedard.

• BuildSlaveSupport/ews-build/steps.py:

(RunWebKitTests.start): Initialize log_observer.
(RunWebKitTests._strip_python_logging_prefix): Copied from similar code in build.webkit.org buildbot config.
(RunWebKitTests._parseRunWebKitTestsOutput): Ditto.
(RunWebKitTests.commandComplete): Gather and parse the stdout and stderr logs.
(RunWebKitTests.evaluateResult): Analyze the results and decide build status.
(RunWebKitTests.getResultSummary): Update build and step summary.

• BuildSlaveSupport/ews-build/steps_unittest.py: Added and updated unit-tests.

Location:

trunk/Tools

Files:

• BuildSlaveSupport/ews-build/steps.py (modified) (3 diffs)
• BuildSlaveSupport/ews-build/steps_unittest.py (modified) (1 diff)
• ChangeLog (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/ews-build/steps.py

@@ -845 +845 @@
 
     def start(self):
+        self.log_observer = logobserver.BufferLogObserver(wantStderr=True)
+        self.addLogObserver('stdio', self.log_observer)
+
+        self.incorrectLayoutLines = []
         platform = self.getProperty('platform')
         appendCustomBuildFlags(self, platform, self.getProperty('fullPlatform'))
@@ -856 +860 @@
         return shell.Test.start(self)
 
+    # FIXME: This will break if run-webkit-tests changes its default log formatter.
+    nrwt_log_message_regexp = re.compile(r'\d{2}:\d{2}:\d{2}(\.\d+)?\s+\d+\s+(?P<message>.*)')
+
+    def _strip_python_logging_prefix(self, line):
+        match_object = self.nrwt_log_message_regexp.match(line)
+        if match_object:
+            return match_object.group('message')
+        return line
+
+    def _parseRunWebKitTestsOutput(self, logText):
+        incorrectLayoutLines = []
+        expressions = [
+            ('flakes', re.compile(r'Unexpected flakiness.+\((\d+)\)')),
+            ('new passes', re.compile(r'Expected to .+, but passed:\s+\((\d+)\)')),
+            ('missing results', re.compile(r'Regressions: Unexpected missing results\s+\((\d+)\)')),
+            ('failures', re.compile(r'Regressions: Unexpected.+\((\d+)\)')),
+        ]
+        testFailures = {}
+
+        for line in logText.splitlines():
+            if line.find('Exiting early') >= 0 or line.find('leaks found') >= 0:
+                incorrectLayoutLines.append(self._strip_python_logging_prefix(line))
+                continue
+            for name, expression in expressions:
+                match = expression.search(line)
+
+                if match:
+                    testFailures[name] = testFailures.get(name, 0) + int(match.group(1))
+                    break
+
+                # FIXME: Parse file names and put them in results
+
+        for name in testFailures:
+            incorrectLayoutLines.append(str(testFailures[name]) + ' ' + name)
+
+        self.incorrectLayoutLines = incorrectLayoutLines
+
+    def commandComplete(self, cmd):
+        shell.Test.commandComplete(self, cmd)
+        logText = self.log_observer.getStdout() + self.log_observer.getStderr()
+        self._parseRunWebKitTestsOutput(logText)
+
+    def evaluateResult(self, cmd):
+        result = SUCCESS
+
+        if self.incorrectLayoutLines:
+            if len(self.incorrectLayoutLines) == 1:
+                line = self.incorrectLayoutLines[0]
+                if line.find('were new') >= 0 or line.find('was new') >= 0 or line.find(' leak') >= 0:
+                    return WARNINGS
+
+            for line in self.incorrectLayoutLines:
+                if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
+                    result = WARNINGS
+                else:
+                    return FAILURE
+
+        # Return code from Tools/Scripts/layout_tests/run_webkit_tests.py.
+        # This means that an exception was raised when running run-webkit-tests and
+        # was never handled.
+        if cmd.rc == 254:
+            return RETRY
+        if cmd.rc != 0:
+            return FAILURE
+
+        return result
+
     def evaluateCommand(self, cmd):
-        rc = super(RunWebKitTests, self).evaluateCommand(cmd)
-        if rc == SUCCESS:
+        rc = self.evaluateResult(cmd)
+        if rc == SUCCESS or rc == WARNINGS:
             message = 'Passed layout tests'
             self.descriptionDone = message
@@ -866 +937 @@
             self.build.addStepsAfterCurrentStep([ArchiveTestResults(), UploadTestResults(), ExtractTestResults(), ReRunWebKitTests()])
         return rc
+
+    def getResultSummary(self):
+        status = self.name
+
+        if self.results != SUCCESS and self.incorrectLayoutLines:
+            status = u' '.join(self.incorrectLayoutLines)
+            return {u'step': status}
+
+        return super(RunWebKitTests, self).getResultSummary()
 
 

trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py

@@ -975 +975 @@
         self.expectOutcome(result=SUCCESS, state_string='Passed layout tests')
         return self.runStep()
+
+    def test_warnings(self):
+        self.setupStep(RunWebKitTests())
+        self.setProperty('fullPlatform', 'ios-simulator')
+        self.setProperty('configuration', 'release')
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        logfiles={'json': self.jsonFileName},
+                        command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
+                        )
+            + 0
+            + ExpectShell.log('stdio', stdout='''Unexpected flakiness: timeouts (2)
+                              imported/blink/storage/indexeddb/blob-valid-before-commit.html [ Timeout Pass ]
+                              storage/indexeddb/modern/deleteindex-2.html [ Timeout Pass ]'''),
+        )
+        self.expectOutcome(result=WARNINGS, state_string='2 flakes')
+        return self.runStep()
+
+    def test_unexpected_error(self):
+        self.setupStep(RunWebKitTests())
+        self.setProperty('fullPlatform', 'mac-highsierra')
+        self.setProperty('configuration', 'debug')
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        logfiles={'json': self.jsonFileName},
+                        command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-new-test-results', '--no-show-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--debug', '--results-directory', 'layout-test-results', '--debug-rwt-logging'],
+                        )
+            + ExpectShell.log('stdio', stdout='Unexpected error.')
+            + 254,
+        )
+        self.expectOutcome(result=RETRY, state_string='layout-tests (retry)')
+        return self.runStep()
+
 
     def test_failure(self):

trunk/Tools/ChangeLog

@@ +1 @@
+2019-07-15  Aakash Jain  <aakash_jain@apple.com>
+
+        [ews-build] Parse and display layout test failures
+        https://bugs.webkit.org/show_bug.cgi?id=199709
+
+        Rubber-stamped by Jonathan Bedard.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (RunWebKitTests.start): Initialize log_observer.
+        (RunWebKitTests._strip_python_logging_prefix): Copied from similar code in build.webkit.org buildbot config.
+        (RunWebKitTests._parseRunWebKitTestsOutput): Ditto.
+        (RunWebKitTests.commandComplete): Gather and parse the stdout and stderr logs.
+        (RunWebKitTests.evaluateResult): Analyze the results and decide build status.
+        (RunWebKitTests.getResultSummary): Update build and step summary.
+        * BuildSlaveSupport/ews-build/steps_unittest.py: Added and updated unit-tests.
+
 2019-07-15  Commit Queue  <commit-queue@webkit.org>