Changeset 268526 in webkit

Timestamp:

Oct 15, 2020 9:27:48 AM (4 years ago)

Author:

aakash_jain@apple.com

Message:

EWS bot should fail if a new test is missing its result
​https://bugs.webkit.org/show_bug.cgi?id=177723

Reviewed by Jonathan Bedard.

• BuildSlaveSupport/ews-build/steps.py:

(RunWebKitTests.evaluateResult):

• BuildSlaveSupport/ews-build/steps_unittest.py: Added unit-test.

Location:

trunk/Tools

Files:

• BuildSlaveSupport/ews-build/steps.py (modified) (1 diff)
• BuildSlaveSupport/ews-build/steps_unittest.py (modified) (2 diffs)
• ChangeLog (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/ews-build/steps.py

@@ -1996 +1996 @@
 
             for line in self.incorrectLayoutLines:
-                if line.find('flakes') >= 0 or line.find('new passes') >= 0 or line.find('missing results') >= 0:
+                if line.find('flakes') >= 0 or line.find('new passes') >= 0:
                     result = WARNINGS
+                elif line.find('missing results') >= 0:
+                    return FAILURE
                 else:
                     return FAILURE

trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py

@@ -1592 +1592 @@
         '''
 
+        self.results_with_missing_results = '''ADD_RESULTS({"tests":{"http":{"wpt":{"css":{"css-highlight-api":{"highlight-image-expected-mismatched.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"},"highlight-image.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"}}}}}}, "interrupted":false});
+        '''
+
         return self.setUpBuildStep()
 
@@ -1738 +1741 @@
         self.assertEqual(self.getProperty(self.property_exceed_failure_limit), False)
         self.assertEqual(self.getProperty(self.property_failures), ['fast/scrolling/ios/reconcile-layer-position-recursive.html'])
+        return rc
+
+    def test_parse_results_json_with_missing_results(self):
+        self.configureStep()
+        self.setProperty('fullPlatform', 'ios-simulator')
+        self.setProperty('configuration', 'release')
+        self.expectRemoteCommands(
+            ExpectShell(workdir='wkdir',
+                        logfiles={'json': self.jsonFileName},
+                        logEnviron=False,
+                        command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging', '--exit-after-n-failures', '30', '--skip-failing-tests'],
+                        )
+            + 2
+            + ExpectShell.log('json', stdout=self.results_with_missing_results),
+        )
+        self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)')
+        rc = self.runStep()
+        self.assertEqual(self.getProperty(self.property_exceed_failure_limit), False)
+        self.assertEqual(self.getProperty(self.property_failures),
+                         ['http/wpt/css/css-highlight-api/highlight-image-expected-mismatched.html',
+                          'http/wpt/css/css-highlight-api/highlight-image.html'])
         return rc
 

trunk/Tools/ChangeLog

@@ +1 @@
+2020-10-15  Aakash Jain  <aakash_jain@apple.com>
+
+        EWS bot should fail if a new test is missing its result
+        https://bugs.webkit.org/show_bug.cgi?id=177723
+
+        Reviewed by Jonathan Bedard.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (RunWebKitTests.evaluateResult):
+        * BuildSlaveSupport/ews-build/steps_unittest.py: Added unit-test.
+
 2020-10-15  Lauro Moura  <lmoura@igalia.com>