Changeset 124967 in webkit

Timestamp:

Aug 7, 2012 6:50:11 PM (12 years ago)

Author:

dpranke@chromium.org

Message:

[NRWT] Would like an output mode similar to ORWT verbose one
​https://bugs.webkit.org/show_bug.cgi?id=88702

Reviewed by Ryosuke Niwa.

Change the --verbose logging for new-run-webkit-tests so that
it matches ORWT more; we just print one line per test. Use
--debug-rwt-logging to get the full debug stream (aka old ORWT
--verbose).

• Scripts/webkitpy/layout_tests/models/test_expectations.py:

(TestExpectations):

• Scripts/webkitpy/layout_tests/run_webkit_tests.py:

(_set_up_derived_options):
(parse_args):

• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

(MainTest.test_retrying_and_flaky_tests):

• Scripts/webkitpy/layout_tests/views/printing.py:

(print_options):
(Printer._print_result_summary_entry):
(Printer._print_one_line_summary):
(Printer._print_test_result):
(Printer._print_baseline):
(Printer._print_unexpected_results):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/models/test_expectations.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/run_webkit_tests.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/views/printing.py (modified) (7 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-08-07  Dirk Pranke  <dpranke@chromium.org>
+
+        [NRWT] Would like an output mode similar to ORWT verbose one
+        https://bugs.webkit.org/show_bug.cgi?id=88702
+
+        Reviewed by Ryosuke Niwa.
+
+        Change the --verbose logging for new-run-webkit-tests so that
+        it matches ORWT more; we just print one line per test. Use
+        --debug-rwt-logging to get the full debug stream (aka old ORWT
+        --verbose).
+
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+        (TestExpectations):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        (_set_up_derived_options):
+        (parse_args):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+        (MainTest.test_retrying_and_flaky_tests):
+        * Scripts/webkitpy/layout_tests/views/printing.py:
+        (print_options):
+        (Printer._print_result_summary_entry):
+        (Printer._print_one_line_summary):
+        (Printer._print_test_result):
+        (Printer._print_baseline):
+        (Printer._print_unexpected_results):
+
 2012-08-07  Dirk Pranke  <dpranke@chromium.org>
 

trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

@@ -691 +691 @@
                     'missing': MISSING}
 
-    EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped'),
-                                PASS: ('pass', 'passes'),
-                                TEXT: ('text diff mismatch',
-                                       'text diff mismatch'),
-                                IMAGE: ('image mismatch', 'image mismatch'),
-                                IMAGE_PLUS_TEXT: ('image and text mismatch',
-                                                  'image and text mismatch'),
-                                AUDIO: ('audio mismatch', 'audio mismatch'),
-                                CRASH: ('crash', 'crashes'),
-                                TIMEOUT: ('test timed out', 'tests timed out'),
-                                MISSING: ('no expected result found',
-                                          'no expected results found')}
+    # (aggregated by category, pass/fail/skip, type)
+    EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped', ''),
+                                PASS: ('passes', 'passed', ''),
+                                TEXT: ('text failures', 'failed', ' (text diff)'),
+                                IMAGE: ('image-only failures', 'failed', ' (image diff)'),
+                                IMAGE_PLUS_TEXT: ('both image and text failures', 'failed', ' (both image and text diffs'),
+                                AUDIO: ('audio failures', 'failed', ' (audio diff)'),
+                                CRASH: ('crashes', 'crashed', ''),
+                                TIMEOUT: ('timeouts', 'timed out', ''),
+                                MISSING: ('missing results', 'is missing an expected result', '')}
 
     EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT, TEXT, IMAGE, AUDIO, SKIP)

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -178 +178 @@
         options.pixel_test_directories = list(varified_dirs)
 
+    if options.run_singly:
+        options.verbose = True
+
     return warnings
 
@@ -390 +393 @@
             help=("Run a the tests in batches (n), after every n tests, "
                   "DumpRenderTree is relaunched."), type="int", default=None),
-        # old-run-webkit-tests has --run-singly imply --verbose.
         optparse.make_option("--run-singly", action="store_true",
-            default=False, help="run a separate DumpRenderTree for each test"),
+            default=False, help="run a separate DumpRenderTree for each test (implies --verbose)"),
         optparse.make_option("--child-processes",
             help="Number of DumpRenderTrees to run in parallel."),

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@ -765 +765 @@
         self.assertTrue('Clobbering old results' in err.getvalue())
         self.assertTrue('flaky/text.html' in err.getvalue())
-        self.assertTrue('Unexpected text diff' in out.getvalue())
+        self.assertTrue('Unexpected text failures' in out.getvalue())
         self.assertFalse('Unexpected flakiness' in out.getvalue())
         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))

trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py

@@ -47 +47 @@
         optparse.make_option('-q', '--quiet', action='store_true', default=False,
                              help='run quietly (errors, warnings, and progress only)'),
-        optparse.make_option('-v', '--verbose', action='store_true', default=False, dest='debug_rwt_logging',
-                             help='same as --debug-rwt-logging (for now)'),
+        optparse.make_option('-v', '--verbose', action='store_true', default=False,
+                             help='print a summarized result for every test (one line per test)'),
         optparse.make_option('--details', action='store_true', default=False,
                              help='print detailed results for every test'),
@@ -294 +294 @@
             if not_passing and len(results):
                 pct = len(results) * 100.0 / not_passing
-                self._print_for_bot("  %5d %-24s (%4.1f%%)" % (len(results), desc[len(results) != 1], pct))
+                self._print_for_bot("  %5d %-24s (%4.1f%%)" % (len(results), desc[0], pct))
 
     def _print_one_line_summary(self, total, expected, unexpected):
@@ -303 +303 @@
             incomplete_str = " (%d didn't run)" % incomplete
 
-        if self._options.debug_rwt_logging or unexpected:
+        if self._options.verbose or self._options.debug_rwt_logging or unexpected:
             self.writeln("")
 
@@ -328 +328 @@
         if self._options.details:
             self._print_test_trace(result, exp_str, got_str)
-        elif not expected:
-            self._print_unexpected_test_result(result)
+        elif (self._options.verbose and not self._options.debug_rwt_logging) or not expected:
+            desc = TestExpectations.EXPECTATION_DESCRIPTIONS[result.type]
+            self.writeln("%s %s%s%s" % (result.test_name, desc[1], "" if expected else " unexpectedly", desc[2]))
 
     def _print_test_trace(self, result, exp_str, got_str):
@@ -356 +357 @@
             relpath = '<none>'
         self._print_default('  %s: %s' % (extension[1:], relpath))
-
-    def _print_unexpected_test_result(self, result):
-        desc = TestExpectations.EXPECTATION_DESCRIPTIONS[result.type][0]
-        self._print_quiet("  %s -> unexpected %s" % (result.test_name, desc))
 
     def _print_progress(self, result_summary, retrying, test_list):
@@ -419 +416 @@
             for key, tests in flaky.iteritems():
                 result = TestExpectations.EXPECTATIONS[key.lower()]
-                self._print_for_bot("Unexpected flakiness: %s (%d)" % (descriptions[result][1], len(tests)))
+                self._print_for_bot("Unexpected flakiness: %s (%d)" % (descriptions[result][0], len(tests)))
                 tests.sort()
 
@@ -436 +433 @@
             for key, tests in regressions.iteritems():
                 result = TestExpectations.EXPECTATIONS[key.lower()]
-                self._print_for_bot("Regressions: Unexpected %s : (%d)" % (descriptions[result][1], len(tests)))
+                self._print_for_bot("Regressions: Unexpected %s : (%d)" % (descriptions[result][0], len(tests)))
                 tests.sort()
                 for test in tests: