Changeset 110724 in webkit

Timestamp:

Mar 14, 2012 11:55:58 AM (12 years ago)

Author:

dpranke@chromium.org

Message:

Please add a way to manually skip some tests in NRWT
​https://bugs.webkit.org/show_bug.cgi?id=81019

Reviewed by Ojan Vafai.

This patch implements the -i / --ignore-tests flag from ORWT
and refactors the test_expectations.py code slightly to handle
it.

• Scripts/webkitpy/layout_tests/controllers/manager.py:

(Manager.parse_expectations):

• Scripts/webkitpy/layout_tests/models/test_expectations.py:

(TestExpectations.init):

• Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:

(test_add_skipped_tests):
(test_add_skipped_tests_duplicate):

• Scripts/webkitpy/layout_tests/run_webkit_tests.py:

(parse_args):

• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

(MainTest.test_ignore_tests):
(MainTest.test_ignore_tests.assert_ignored):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/controllers/manager.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/models/test_expectations.py (modified) (3 diffs)
• Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-03-14  Dirk Pranke  <dpranke@chromium.org>
+
+        Please add a way to manually skip some tests in NRWT
+        https://bugs.webkit.org/show_bug.cgi?id=81019
+
+        Reviewed by Ojan Vafai.
+
+        This patch implements the -i / --ignore-tests flag from ORWT
+        and refactors the test_expectations.py code slightly to handle
+        it.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (Manager.parse_expectations):
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+        (TestExpectations.__init__):
+        * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:
+        (test_add_skipped_tests):
+        (test_add_skipped_tests_duplicate):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        (parse_args):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+        (MainTest.test_ignore_tests):
+        (MainTest.test_ignore_tests.assert_ignored):
+
 2012-03-14  Dirk Pranke  <dpranke@chromium.org>
 

trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@ -365 +365 @@
         invalid syntax."""
         port = self._port
+        tests_to_ignore = set(self._options.ignore_tests)
         self._expectations = test_expectations.TestExpectations(
             port,
@@ -371 +372 @@
             port.test_configuration(),
             self._options.lint_test_files,
-            port.test_expectations_overrides())
+            port.test_expectations_overrides(),
+            port.skipped_tests(self._test_files).union(tests_to_ignore))
 
     def _split_into_chunks_if_necessary(self, skipped):

trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

@@ -688 +688 @@
 
     def __init__(self, port, tests, expectations,
-                 test_config, is_lint_mode=False, overrides=None):
+                 test_config, is_lint_mode=False, overrides=None,
+                 skipped_tests=None):
         """Loads and parses the test expectations given in the string.
         Args:
@@ -703 +704 @@
                 that need to manage two sets of expectations (e.g., upstream
                 and downstream expectations).
+            skipped_tests: test paths to skip.
         """
         self._full_test_list = tests
@@ -714 +716 @@
         self._expectations = self._parser.parse(expectations)
         self._add_expectations(self._expectations, overrides_allowed=False)
-        self._add_skipped_tests(port.skipped_tests(tests))
+        self._add_skipped_tests(skipped_tests or [])
 
         if overrides:

trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py

@@ -263 +263 @@
     def test_add_skipped_tests(self):
         port = MockHost().port_factory.get('qt')
-        port._filesystem.files[port._filesystem.join(port.layout_tests_dir(), 'platform/qt/Skipped')] = 'failures/expected/text.html'
         port._filesystem.files[port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html')] = 'foo'
-        expectations = TestExpectations(port, tests=['failures/expected/text.html'], expectations='', test_config=port.test_configuration())
+        expectations = TestExpectations(port, tests=['failures/expected/text.html'], expectations='', test_config=port.test_configuration(), skipped_tests=set(['failures/expected/text.html']))
         self.assertEquals(expectations.get_modifiers('failures/expected/text.html'), [TestExpectationParser.DUMMY_BUG_MODIFIER, TestExpectationParser.SKIP_MODIFIER])
         self.assertEquals(expectations.get_expectations('failures/expected/text.html'), set([PASS]))
@@ -271 +270 @@
     def test_add_skipped_tests_duplicate(self):
         port = MockHost().port_factory.get('qt')
-        port._filesystem.files[port._filesystem.join(port.layout_tests_dir(), 'platform/qt/Skipped')] = 'failures/expected/text.html'
         port._filesystem.files[port._filesystem.join(port.layout_tests_dir(), 'failures/expected/text.html')] = 'foo'
-        self.assertRaises(ParseError, TestExpectations, port, tests=['failures/expected/text.html'], expectations='BUGX : failures/expected/text.html = text\n', test_config=port.test_configuration(), is_lint_mode=True)
+        self.assertRaises(ParseError, TestExpectations, port, tests=['failures/expected/text.html'], expectations='BUGX : failures/expected/text.html = text\n', test_config=port.test_configuration(), is_lint_mode=True, skipped_tests=set(['failures/expected/text.html']))
 
 

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -380 +380 @@
         # -i|--ignore-tests               Comma-separated list of directories
         #                                 or tests to ignore
+        optparse.make_option("-i", "--ignore-tests", action="append", default=[],
+            help="directories or test to ignore (may specify multiple times)"),
         optparse.make_option("--test-list", action="append",
             help="read list of tests to run from file", metavar="FILE"),

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@ -398 +398 @@
         self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html'])
 
+    def test_ignore_tests(self):
+        def assert_ignored(args, tests_expected_to_run):
+            tests_to_run = ['failures/expected/image.html', 'passes/image.html']
+            tests_run = get_tests_run(args + tests_to_run, tests_included=True, flatten_batches=True)
+            self.assertEquals(tests_run, tests_expected_to_run)
+
+        assert_ignored(['-i', 'failures/expected/image.html'], ['passes/image.html'])
+        assert_ignored(['-i', 'passes'], ['failures/expected/image.html'])
+
+        # Note here that there is an expectation for failures/expected/image.html already, but
+        # it is overriden by the command line arg. This might be counter-intuitive.
+        # FIXME: This isn't currently working ...
+        # assert_ignored(['-i', 'failures/expected'], ['passes/image.html'])
+
     def test_iterations(self):
         tests_to_run = ['passes/image.html', 'passes/text.html']