Changeset 120348 in webkit

Timestamp:

Jun 14, 2012 12:40:06 PM (12 years ago)

Author:

dpranke@chromium.org

Message:

NRWT should honor --skipped=[default|ignore|only], like ORWT does
​https://bugs.webkit.org/show_bug.cgi?id=66308

Reviewed by Ryosuke Niwa.

This patch adds support for ORWT's --skipped=(default|ignore|only)
flag and cleans up the interaction between that and --ignore.

Individual tests (but not directories) explicitly listed on the
command line will always be run regardless of what is passed
for --skipped and --ignore.

This patch also changes the "found" and "running" log messages
since it wasn't clear how the skipped tests were included in those numbers.

• Scripts/webkitpy/layout_tests/controllers/manager.py:

(Manager.init):
(Manager.collect_tests):
(Manager.prepare_lists_and_print_output):

• Scripts/webkitpy/layout_tests/port/test.py:
• Scripts/webkitpy/layout_tests/run_webkit_tests.py:

(_set_up_derived_options):
(parse_args):

• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

(MainTest.test_ignore_flag):
(MainTest):
(MainTest.test_skipped_flag):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/controllers/manager.py (modified) (4 diffs)
• Scripts/webkitpy/layout_tests/port/test.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-06-14  Dirk Pranke  <dpranke@chromium.org>
+
+        NRWT should honor --skipped=[default|ignore|only], like ORWT does
+        https://bugs.webkit.org/show_bug.cgi?id=66308
+
+        Reviewed by Ryosuke Niwa.
+
+        This patch adds support for ORWT's --skipped=(default|ignore|only)
+        flag and cleans up the interaction between that and --ignore.
+
+        Individual tests (but not directories) explicitly listed on the
+        command line will always be run regardless of what is passed
+        for --skipped and --ignore.
+
+        This patch also changes the "found" and "running" log messages
+        since it wasn't clear how the skipped tests were included in those numbers.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (Manager.__init__):
+        (Manager.collect_tests):
+        (Manager.prepare_lists_and_print_output):
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        (_set_up_derived_options):
+        (parse_args):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+        (MainTest.test_ignore_flag):
+        (MainTest):
+        (MainTest.test_skipped_flag):
+
 2012-06-14  Takashi Toyoshima  <toyoshim@chromium.org>
 

trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@ -317 +317 @@
         # a set of test files, and the same tests as a list
 
+        self._paths = set()
+
         # FIXME: Rename to test_names.
         self._test_files = set()
@@ -341 +343 @@
         if self._options.test_list:
             paths += self._strip_test_dir_prefixes(read_test_files(self._filesystem, self._options.test_list, self._port.TEST_PATH_SEPARATOR))
+        self._paths = set(paths)
         self._test_files = self._port.tests(paths)
 
@@ -457 +460 @@
         # Remove skipped - both fixable and ignored - files from the
         # top-level list of files to test.
+        found_test_files = set(self._test_files)
         num_all_test_files = len(self._test_files)
-        self._printer.print_expected("Found:  %d tests" % (len(self._test_files)))
+
+        skipped = self._expectations.get_tests_with_result_type(test_expectations.SKIP)
+        if not self._options.http:
+            skipped.update(set(self._http_tests()))
+
+        if self._options.skipped == 'only':
+            self._test_files = self._test_files.intersection(skipped)
+        elif self._options.skipped == 'default':
+            self._test_files -= skipped
+        elif self._options.skipped == 'ignore':
+            pass  # just to be clear that we're ignoring the skip list.
+
+        if self._options.skip_failing_tests:
+            self._test_files -= self._expectations.get_tests_with_result_type(test_expectations.FAIL)
+            self._test_files -= self._expectations.get_tests_with_result_type(test_expectations.FLAKY)
+
+        # now make sure we're explicitly running any tests passed on the command line.
+        self._test_files.update(found_test_files.intersection(self._paths))
+
         if not num_all_test_files:
             _log.critical('No tests to run.')
             return None
 
-        skipped = set()
-
-        if not self._options.http:
-            skipped = skipped.union(self._http_tests())
-
-        if num_all_test_files > 1 and not self._options.force:
-            skipped.update(self._expectations.get_tests_with_result_type(test_expectations.SKIP))
-            if self._options.skip_failing_tests:
-                skipped.update(self._expectations.get_tests_with_result_type(test_expectations.FAIL))
-                skipped.update(self._expectations.get_tests_with_result_type(test_expectations.FLAKY))
-
-        self._test_files -= skipped
+        num_skipped = num_all_test_files - len(self._test_files)
+        if num_skipped:
+            self._printer.print_expected("Running %s (found %d, skipping %d)." % (
+                grammar.pluralize('test', num_all_test_files - num_skipped),
+                num_all_test_files, num_skipped))
+        elif len(self._test_files) > 1:
+            self._printer.print_expected("Running all %d tests." % len(self._test_files))
+        else:
+            self._printer.print_expected("Running %1 test.")
 
         # Create a sorted list of test files so the subset chunk,
@@ -505 +524 @@
         self._print_expected_results_of_type(result_summary, test_expectations.SKIP, "skipped")
 
-        if self._options.force:
-            self._printer.print_expected('Running all tests, including skips (--force)')
-        else:
+        if self._options.skipped != 'ignore':
             # Note that we don't actually run the skipped tests (they were
             # subtracted out of self._test_files, above), but we stub out the

trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py

@@ -175 +175 @@
               expected_checksum=None,
               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
+    tests.add('passes/skipped/skip.html')
 
     # Note that here the checksums don't match but the images do, so this test passes "unexpectedly".
@@ -264 +265 @@
 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
 WONTFIX SKIP : failures/expected/exception.html = CRASH
+WONTFIX SKIP : passes/skipped/skip.html = PASS
 """)
 

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -158 +158 @@
         options.additional_platform_directory = normalized_platform_directories
 
-    if not options.http and options.force:
-        warnings.append("--no-http is ignored since --force is also provided")
+    if not options.http and options.skipped in ('ignore', 'only'):
+        warnings.append("--force/--skipped=%s overrides --no-http." % (options.skipped))
         options.http = True
 
@@ -348 +348 @@
                  "DumpRenderTree; option is split on whitespace before "
                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
-        # old-run-webkit-tests:
-        # -i|--ignore-tests               Comma-separated list of directories
-        #                                 or tests to ignore
         optparse.make_option("-i", "--ignore-tests", action="append", default=[],
             help="directories or test to ignore (may specify multiple times)"),
         optparse.make_option("--test-list", action="append",
             help="read list of tests to run from file", metavar="FILE"),
-        # old-run-webkit-tests uses --skipped==[default|ignore|only]
-        # instead of --force:
-        optparse.make_option("--force", action="store_true", default=False,
-            help="Run all tests, even those marked SKIP in the test list"),
+        optparse.make_option("--skipped", action="store", default="default",
+            help="control how tests marked SKIP are run. 'default' == Skip, 'ignore' == Run them anyway, 'only' == only run the SKIP tests."),
+        optparse.make_option("--force", dest="skipped", action="store_const", const='ignore',
+            help="Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)"),
         optparse.make_option("--time-out-ms",
             help="Set the timeout for each test"),

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@ -402 +402 @@
         self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html'])
 
-    def test_ignore_tests(self):
-        def assert_ignored(args, tests_expected_to_run):
-            tests_to_run = ['failures/expected/image.html', 'passes/image.html']
-            tests_run = get_tests_run(args + tests_to_run, tests_included=True, flatten_batches=True)
-            self.assertEquals(tests_run, tests_expected_to_run)
-
-        assert_ignored(['-i', 'failures/expected/image.html'], ['passes/image.html'])
-        assert_ignored(['-i', 'passes'], ['failures/expected/image.html'])
-
-        # Note here that there is an expectation for failures/expected/image.html already, but
-        # it is overriden by the command line arg. This might be counter-intuitive.
-        # FIXME: This isn't currently working ...
-        # assert_ignored(['-i', 'failures/expected'], ['passes/image.html'])
+    def test_ignore_flag(self):
+        # Note that passes/image.html is expected to be run since we specified it directly.
+        tests_run = get_tests_run(['-i', 'passes', 'passes/image.html'], flatten_batches=True, tests_included=True)
+        self.assertFalse('passes/text.html' in tests_run)
+        self.assertTrue('passes/image.html' in tests_run)
+
+    def test_skipped_flag(self):
+        tests_run = get_tests_run(['passes'], tests_included=True, flatten_batches=True)
+        self.assertFalse('passes/skipped/skip.html' in tests_run)
+        num_tests_run_by_default = len(tests_run)
+
+        # Check that nothing changes when we specify skipped=default.
+        self.assertEquals(len(get_tests_run(['--skipped=default', 'passes'], tests_included=True, flatten_batches=True)),
+                          num_tests_run_by_default)
+
+        # Now check that we run one more test (the skipped one).
+        tests_run = get_tests_run(['--skipped=ignore', 'passes'], tests_included=True, flatten_batches=True)
+        self.assertTrue('passes/skipped/skip.html' in tests_run)
+        self.assertEquals(len(tests_run), num_tests_run_by_default + 1)
+
+        # Now check that we only run the skipped test.
+        self.assertEquals(get_tests_run(['--skipped=only', 'passes'], tests_included=True, flatten_batches=True),
+                          ['passes/skipped/skip.html'])
 
     def test_iterations(self):