Changeset 235771 in webkit

Timestamp:

Sep 6, 2018 5:24:51 PM (6 years ago)

Author:

Simon Fraser

Message:

Log when leak detection changes the test result
​https://bugs.webkit.org/show_bug.cgi?id=189293

Reviewed by Jon Lee.

When you have an "Leak" expectation in TestExpectations, it's confusing to see a test unexpectedly pass,
but then show up as an expected fail at the end (because leak detection happens at the end of a shard).
So log when leak detection changes a test result.

• Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:

(LayoutTestRunner._annotate_results_with_additional_failures):

• Scripts/webkitpy/layout_tests/models/test_expectations.py:

(TestExpectations):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/models/test_expectations.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/views/printing.py (modified) (2 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2018-09-06  Simon Fraser  <simon.fraser@apple.com>
+
+        Log when leak detection changes the test result
+        https://bugs.webkit.org/show_bug.cgi?id=189293
+
+        Reviewed by Jon Lee.
+        
+        When you have an "Leak" expectation in TestExpectations, it's confusing to see a test unexpectedly pass,
+        but then show up as an expected fail at the end (because leak detection happens at the end of a shard).
+        So log when leak detection changes a test result.
+
+        * Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
+        (LayoutTestRunner._annotate_results_with_additional_failures):
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+        (TestExpectations):
+
 2018-09-06  Wenson Hsieh  <wenson_hsieh@apple.com>
 

trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py

@@ -203 +203 @@
                 was_expected = self._expectations.matches_an_expected_result(new_result.test_name, existing_result.type, expectations)
                 now_expected = self._expectations.matches_an_expected_result(new_result.test_name, new_result.type, expectations)
+                if was_expected != now_expected:
+                    # When annotation is not just about leaks, this logging should be changed.
+                    _log.warning('  %s -> changed by leak detection from a %s (%s) to a %s (%s)' % (new_result.test_name,
+                        TestExpectations.EXPECTATION_DESCRIPTION[existing_result.type], 'expected' if was_expected else 'unexpected',
+                        TestExpectations.EXPECTATION_DESCRIPTION[new_result.type], 'expected' if now_expected else 'unexpected'))
                 run_results.change_result_to_failure(existing_result, new_result, was_expected, now_expected)
 

trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

@@ -816 +816 @@
                     'skip': SKIP}
 
+    # Singulars
+    EXPECTATION_DESCRIPTION = {SKIP: 'skipped',
+                                PASS: 'pass',
+                                FAIL: 'failure',
+                                IMAGE: 'image-only failure',
+                                TEXT: 'text-only failure',
+                                IMAGE_PLUS_TEXT: 'image and text failure',
+                                AUDIO: 'audio failure',
+                                CRASH: 'crash',
+                                TIMEOUT: 'timeout',
+                                MISSING: 'missing',
+                                LEAK: 'leak'}
+
     # (aggregated by category, pass/fail/skip, type)
     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',

trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py

@@ -322 +322 @@
         test_name = result.test_name
 
-        result_message = self._result_message(result.type, result.failures, expected, self._options.verbose)
+        result_message = self._result_message(result.type, result.failures, expected, exp_str, self._options.verbose)
 
         if self._options.details:
@@ -341 +341 @@
         self._running_tests.remove(test_name)
 
-    def _result_message(self, result_type, failures, expected, verbose):
-        exp_string = ' unexpectedly' if not expected else ''
+    def _result_message(self, result_type, failures, expected, exp_str, verbose):
+        exp_string = ''
+        if not expected:
+            exp_string = ' (leak detection is pending)' if 'LEAK' in exp_str else ' unexpectedly'
+
         if result_type == test_expectations.PASS:
             return ' passed%s' % exp_string