Changeset 116552 in webkit

Timestamp:

May 9, 2012 12:59:44 PM (12 years ago)

Author:

dpranke@chromium.org

Message:

nrwt: --skip-failing-tests should also skip flaky tests
​https://bugs.webkit.org/show_bug.cgi?id=85941

Reviewed by Ojan Vafai.

This changes --skip-failing-tests so that we skip tests that
are believed to pass sometimes as well as tests that are
believed to always fail.

In addition, in testing this I discovered that we had lousy
test coverage of the details of retrying test failures and
--clobber-old-results, so I wrote some more tests.

• Scripts/webkitpy/layout_tests/controllers/manager.py:

(Manager.prepare_lists_and_print_output):

• Scripts/webkitpy/layout_tests/port/test.py:

(unit_test_list):
(TestPort.init):
(TestDriver.run_test):

• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

(MainTest.test_skip_failing_tests):
(MainTest.test_retrying_and_flaky_tests):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/controllers/manager.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/test.py (modified) (3 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (modified) (2 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-05-09  Dirk Pranke  <dpranke@chromium.org>
+
+        nrwt: --skip-failing-tests should also skip flaky tests
+        https://bugs.webkit.org/show_bug.cgi?id=85941
+
+        Reviewed by Ojan Vafai.
+
+        This changes --skip-failing-tests so that we skip tests that
+        are believed to pass sometimes as well as tests that are
+        believed to always fail.
+
+        In addition, in testing this I discovered that we had lousy
+        test coverage of the details of retrying test failures and
+        --clobber-old-results, so I wrote some more tests.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (Manager.prepare_lists_and_print_output):
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        (unit_test_list):
+        (TestPort.__init__):
+        (TestDriver.run_test):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+        (MainTest.test_skip_failing_tests):
+        (MainTest.test_retrying_and_flaky_tests):
+
 2012-05-09  Christophe Dumez  <christophe.dumez@intel.com>
 

trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@ -479 +479 @@
 
         if num_all_test_files > 1 and not self._options.force:
-            skipped = skipped.union(self._expectations.get_tests_with_result_type(test_expectations.SKIP))
+            skipped.update(self._expectations.get_tests_with_result_type(test_expectations.SKIP))
             if self._options.skip_failing_tests:
-                failing = self._expectations.get_tests_with_result_type(test_expectations.FAIL)
-                self._test_files -= failing
+                skipped.update(self._expectations.get_tests_with_result_type(test_expectations.FAIL))
+                skipped.update(self._expectations.get_tests_with_result_type(test_expectations.FLAKY))
 
         self._test_files -= skipped

trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py

@@ -137 +137 @@
     tests.add('failures/expected/text.html', actual_text='text_fail-png')
     tests.add('failures/expected/skip_text.html', actual_text='text diff')
+    tests.add('failures/flaky/text.html')
     tests.add('failures/unexpected/missing_text.html', expected_text=None)
     tests.add('failures/unexpected/missing_image.html', expected_image=None)
@@ -335 +336 @@
         Port.__init__(self, host, port_name, **kwargs)
         self._tests = unit_test_list()
+        self._flakes = set()
         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
         self._results_directory = None
@@ -514 +516 @@
         audio = None
         actual_text = test.actual_text
+
+        if 'flaky' in test_name and not test_name in self._port._flakes:
+            self._port._flakes.add(test_name)
+            actual_text = 'flaky text failure'
+
         if actual_text and test_args and test_name == 'passes/args.html':
             actual_text = actual_text + ' ' + ' '.join(test_args)

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@ -461 +461 @@
 
     def test_skip_failing_tests(self):
-        batches = get_tests_run(['--skip-failing-tests'])
+        # This tests that we skip both known failing and known flaky tests. Because there are
+        # no known flaky tests in the default test_expectations, we add additional expectations.
+        host = MockHost()
+        host.filesystem.write_text_file('/tmp/overrides.txt', 'BUGX : passes/image.html = IMAGE PASS\n')
+
+        batches = get_tests_run(['--skip-failing-tests', '--additional-expectations', '/tmp/overrides.txt'], host=host)
         has_passes_text = False
         for batch in batches:
             self.assertFalse('failures/expected/text.html' in batch)
+            self.assertFalse('passes/image.html' in batch)
             has_passes_text = has_passes_text or ('passes/text.html' in batch)
         self.assertTrue(has_passes_text)
@@ -719 +725 @@
         self.assertEqual(user.opened_urls, [path.abspath_to_uri('/tmp/cwd/foo/results.html')])
 
-    def test_retries_directory(self):
-        host = MockHost()
-        res, out, err, user = logging_run(tests_included=True, host=host)
+    def test_retrying_and_flaky_tests(self):
+        host = MockHost()
+        res, out, err, _ = logging_run(['failures/flaky'], tests_included=True, host=host)
+        self.assertEquals(res, 0)
+        self.assertTrue('Retrying' in err.getvalue())
+        self.assertTrue('Unexpected flakiness' in out.getvalue())
+        self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/tests_run0.txt'))
+        self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/retries/failures/flaky/text-actual.txt'))
+
+        # Now we test that --clobber-old-results does remove the old entries and the old retries,
+        # and that we don't retry again.
+        res, out, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host)
+        self.assertEquals(res, 1)
+        self.assertTrue('Clobbering old results' in err.getvalue())
+        self.assertTrue('flaky/text.html' in err.getvalue())
+        self.assertTrue('Unexpected text diff' in out.getvalue())
+        self.assertFalse('Unexpected flakiness' in out.getvalue())
+        self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
+        self.assertFalse(host.filesystem.exists('retries'))
+
 
     # These next tests test that we run the tests in ascending alphabetical