Changeset 267493 in webkit

Timestamp:

Sep 23, 2020, 12:17:20 PM (6 years ago)

Author:

aakash_jain@apple.com

Message:

Limit number of emails to send for flaky and pre-existing API and layout test failures
​https://bugs.webkit.org/show_bug.cgi?id=216876

Reviewed by Jonathan Bedard.

• BuildSlaveSupport/ews-build/steps.py:

(ReRunWebKitTests): Simply limit the flaky failures immediately after they are calculated, since we don't display
the count of flaky failures. Flaky failures are displayed only to help bot-watchers in noticing flaky tests.
(AnalyzeLayoutTestsResults.report_pre_existing_failures): Limit pre-existing and flaky test failures just before sending emails.
(AnalyzeAPITestsResults.analyzeResults):

Location:

trunk/Tools

Files:

• BuildSlaveSupport/ews-build/steps.py (modified) (6 diffs)
• ChangeLog (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/ews-build/steps.py

@@ -2042 +2042 @@
 class ReRunWebKitTests(RunWebKitTests):
     name = 're-run-layout-tests'
+    NUM_FAILURES_TO_DISPLAY = 10
 
     def evaluateCommand(self, cmd):
@@ -2051 +2052 @@
         tests_that_consistently_failed = first_results_failing_tests.intersection(second_results_failing_tests)
         flaky_failures = first_results_failing_tests.union(second_results_failing_tests) - first_results_failing_tests.intersection(second_results_failing_tests)
+        flaky_failures = list(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY]
         flaky_failures_string = ', '.join(flaky_failures)
 
@@ -2165 +2167 @@
             if len(clean_tree_failures) > self.NUM_FAILURES_TO_DISPLAY:
                 message += ' ...'
-            for clean_tree_failure in clean_tree_failures:
+            for clean_tree_failure in list(clean_tree_failures)[:self.NUM_FAILURES_TO_DISPLAY]:
                 self.send_email_for_pre_existing_failure(clean_tree_failure)
 
@@ -2174 +2176 @@
             if len(flaky_failures) > self.NUM_FAILURES_TO_DISPLAY:
                 message += ' ...'
-            for flaky_failure in flaky_failures:
+            for flaky_failure in list(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY]:
                 self.send_email_for_flaky_failure(flaky_failure)
 
@@ -2590 +2592 @@
         failures_with_patch = first_run_failures.intersection(second_run_failures)
         flaky_failures = first_run_failures.union(second_run_failures) - first_run_failures.intersection(second_run_failures)
+        flaky_failures = list(flaky_failures)[:self.NUM_API_FAILURES_TO_DISPLAY]
         flaky_failures_string = ', '.join(flaky_failures)
         new_failures = failures_with_patch - clean_tree_failures
@@ -2619 +2622 @@
             if clean_tree_failures:
                 message = 'Found {} pre-existing API test failure{}: {}'.format(len(clean_tree_failures), pluralSuffix, clean_tree_failures_string)
-                for clean_tree_failure in clean_tree_failures:
+                for clean_tree_failure in clean_tree_failures_to_display:
                     self.send_email_for_pre_existing_failure(clean_tree_failure)
             if len(clean_tree_failures) > self.NUM_API_FAILURES_TO_DISPLAY:

trunk/Tools/ChangeLog

@@ +1 @@
+2020-09-23  Aakash Jain  <aakash_jain@apple.com>
+
+        Limit number of emails to send for flaky and pre-existing API and layout test failures
+        https://bugs.webkit.org/show_bug.cgi?id=216876
+
+        Reviewed by Jonathan Bedard.
+
+        * BuildSlaveSupport/ews-build/steps.py:
+        (ReRunWebKitTests): Simply limit the flaky failures immediately after they are calculated, since we don't display
+        the count of flaky failures. Flaky failures are displayed only to help bot-watchers in noticing flaky tests.
+        (AnalyzeLayoutTestsResults.report_pre_existing_failures): Limit pre-existing and flaky test failures just before sending emails.
+        (AnalyzeAPITestsResults.analyzeResults):
+
 2020-09-23  Aakash Jain  <aakash_jain@apple.com>