Changeset 90055 in webkit

Timestamp:

Jun 29, 2011 3:25:58 PM (13 years ago)

Author:

abarth@webkit.org

Message:

2011-06-29 Adam Barth <​abarth@webkit.org>

Reviewed by Dirk Pranke.

new-run-webkit-tests complains about missing pixel results instead of plopping down new expectations
​https://bugs.webkit.org/show_bug.cgi?id=38063

This patch changes new-run-webkit-tests to match old-run-webkit-tests
in generating new expected results when the expected results are
missing.

There are still a couple details that are different:

1) Image baselines aren't generated unless you run with -p.
2) Render tree dumps are places in the cross-platform directory instead
of the platform-specific directory.

I'm inclined to deal with both of these issues in follow-up patches.

• Scripts/webkitpy/common/net/layouttestresults.py:
• Scripts/webkitpy/layout_tests/layout_package/manager.py:
• Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
• Scripts/webkitpy/layout_tests/layout_package/test_results.py:
• Scripts/webkitpy/layout_tests/port/test.py:
• Scripts/webkitpy/layout_tests/run_webkit_tests.py:
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/common/net/layouttestresults.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/layout_package/manager.py (modified) (5 diffs)
• Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py (modified) (3 diffs)
• Scripts/webkitpy/layout_tests/layout_package/test_results.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/test.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/run_webkit_tests.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (modified) (3 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-06-29  Adam Barth  <abarth@webkit.org>
+
+        Reviewed by Dirk Pranke.
+
+        new-run-webkit-tests complains about missing pixel results instead of plopping down new expectations
+        https://bugs.webkit.org/show_bug.cgi?id=38063
+
+        This patch changes new-run-webkit-tests to match old-run-webkit-tests
+        in generating new expected results when the expected results are
+        missing.
+
+        There are still a couple details that are different:
+
+        1) Image baselines aren't generated unless you run with -p.
+        2) Render tree dumps are places in the cross-platform directory instead
+        of the platform-specific directory.
+
+        I'm inclined to deal with both of these issues in follow-up patches.
+
+        * Scripts/webkitpy/common/net/layouttestresults.py:
+        * Scripts/webkitpy/layout_tests/layout_package/manager.py:
+        * Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
+        * Scripts/webkitpy/layout_tests/layout_package/test_results.py:
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+
 2011-06-29  Adam Roben  <aroben@apple.com>
 

trunk/Tools/Scripts/webkitpy/common/net/layouttestresults.py

@@ -162 +162 @@
 
     def results_matching_failure_types(self, failure_types):
-        return [result for result in self._test_results if result.has_failure_matching_types(failure_types)]
+        return [result for result in self._test_results if result.has_failure_matching_types(*failure_types)]
 
     def tests_matching_failure_types(self, failure_types):

trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py

@@ -796 +796 @@
         # We exclude the crashes from the list of results to retry, because
         # we want to treat even a potentially flaky crash as an error.
-        failures = self._get_failures(result_summary, include_crashes=False)
+        failures = self._get_failures(result_summary, include_crashes=False, include_missing=False)
         retry_summary = result_summary
         while (len(failures) and self._options.retry_failures and
@@ -807 +807 @@
             # Note that we intentionally ignore the return value here.
             self._run_tests(failures.keys(), retry_summary)
-            failures = self._get_failures(retry_summary, include_crashes=True)
+            failures = self._get_failures(retry_summary, include_crashes=True, include_missing=True)
 
         end_time = time.time()
@@ -934 +934 @@
                 self._fs.rmtree(self._fs.join(self._results_directory, dirname))
 
-    def _get_failures(self, result_summary, include_crashes):
+    def _get_failures(self, result_summary, include_crashes, include_missing):
         """Filters a dict of results and returns only the failures.
 
@@ -949 +949 @@
         for test, result in result_summary.unexpected_results.iteritems():
             if (result.type == test_expectations.PASS or
-                result.type == test_expectations.CRASH and not include_crashes):
+                (result.type == test_expectations.CRASH and not include_crashes) or
+                (result.type == test_expectations.MISSING and not include_missing)):
                 continue
             failed_results[test] = result.type
@@ -1298 +1299 @@
             test_files = result_summary.failures.keys()
         else:
-            unexpected_failures = self._get_failures(result_summary, include_crashes=True)
+            unexpected_failures = self._get_failures(result_summary, include_crashes=True, include_missing=True)
             test_files = unexpected_failures.keys()
 

trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py

@@ -129 +129 @@
         expected_driver_output = self._expected_driver_output()
         test_result = self._compare_output(driver_output, expected_driver_output)
-        test_result_writer.write_test_result(self._port, self._filename,
-                                             driver_output, expected_driver_output, test_result.failures)
+        if self._options.new_test_results:
+            self._add_missing_baselines(test_result, driver_output)
+        test_result_writer.write_test_result(self._port, self._filename, driver_output, expected_driver_output, test_result.failures)
         return test_result
 
@@ -136 +137 @@
         driver_output = self._driver.run_test(self._driver_input())
         failures = self._handle_error(driver_output)
-        test_result_writer.write_test_result(self._port, self._filename,
-                                             driver_output, None, failures)
+        test_result_writer.write_test_result(self._port, self._filename, driver_output, None, failures)
         # FIXME: It the test crashed or timed out, it might be bettter to avoid
         # to write new baselines.
-        self._save_baselines(driver_output)
+        self._overwrite_baselines(driver_output)
         return TestResult(self._filename, failures, driver_output.test_time, driver_output.has_stderr())
 
-    def _save_baselines(self, driver_output):
+    def _add_missing_baselines(self, test_result, driver_output):
+        if test_result.has_failure_matching_types(test_failures.FailureMissingResult):
+            # FIXME: We seem to be putting new text results in non-platform
+            # specific directories even when they're rendertree dumps. Maybe
+            # we should have a different kind of failure for render tree dumps
+            # than for text tests?
+            self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=False)
+        if test_result.has_failure_matching_types(test_failures.FailureMissingAudio):
+            self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=False)
+        if test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash):
+            self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=False)
+
+    def _overwrite_baselines(self, driver_output):
         # Although all DumpRenderTree output should be utf-8,
         # we do not ever decode it inside run-webkit-tests.  For some tests
         # DumpRenderTree may not output utf-8 text (e.g. webarchives).
-        self._save_baseline_data(driver_output.text, ".txt",
-                                 generate_new_baseline=self._options.new_baseline)
-        self._save_baseline_data(driver_output.audio, '.wav',
-                                 generate_new_baseline=self._options.new_baseline)
+        self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=self._options.new_baseline)
+        self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=self._options.new_baseline)
         if self._options.pixel_tests:
-            self._save_baseline_data(driver_output.image, ".png",
-                                     generate_new_baseline=self._options.new_baseline)
+            self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=self._options.new_baseline)
 
     def _save_baseline_data(self, data, modifier, generate_new_baseline=True):
@@ -179 +188 @@
             fs.maybe_make_directory(output_dir)
             output_path = fs.join(output_dir, output_file)
-            _log.debug('writing new baseline result "%s"' % (output_path))
+            _log.debug('Writing new baseline result "%s"' % output_path)
         else:
             output_path = port.expected_filename(self._filename, modifier)
-            _log.debug('resetting baseline result "%s"' % output_path)
+            _log.debug('Resetting baseline result "%s"' % output_path)
 
         port.update_baseline(output_path, data)

trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py

@@ -56 +56 @@
         return not (self == other)
 
-    def has_failure_matching_types(self, types):
+    def has_failure_matching_types(self, *args, **kargs):
         for failure in self.failures:
-            if type(failure) in types:
+            if type(failure) in args:
                 return True
         return False

trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py

@@ -137 +137 @@
               expected_text="foo\r\r\r\n", actual_text="foo\n")
     tests.add('failures/expected/text.html', actual_text='text_fail-png')
+    tests.add('failures/unexpected/missing_text.html', expected_text=None)
+    tests.add('failures/unexpected/missing_image.html', expected_image=None)
     tests.add('failures/unexpected/crash.html', crash=True)
     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -280 +280 @@
 
     old_run_webkit_tests_compat = [
-        # NRWT doesn't generate results by default anyway.
-        _compat_shim_option("--no-new-test-results"),
         # NRWT doesn't sample on timeout yet anyway.
         _compat_shim_option("--no-sample-on-timeout"),
@@ -308 +306 @@
             default=False, help="Reset any existing baselines to the "
                  "generated results"),
+        optparse.make_option("--no-new-test-results", action="store_false",
+            dest="new_test_results", default=True,
+            help="Don't create new baselines when no expected results exist"),
         optparse.make_option("--skip-failing-tests", action="store_true",
             default=False, help="Skip tests that are expected to fail. "

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@ -642 +642 @@
 
 class RebaselineTest(unittest.TestCase):
-    def assertBaselines(self, file_list, file):
+    def assertBaselines(self, file_list, file, extensions):
         "assert that the file_list contains the baselines."""
-        for ext in (".txt", ".png"):
+        for ext in extensions:
             baseline = file + "-expected" + ext
             self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
@@ -663 +663 @@
         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
         self.assertEqual(len(file_list), 4)
-        self.assertBaselines(file_list, "/passes/image")
-        self.assertBaselines(file_list, "/failures/expected/missing_image")
+        self.assertBaselines(file_list, "/passes/image", [".txt", ".png"])
+        self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"])
+
+    def test_missing_results(self):
+        # Test that we update expectations in place. If the expectation
+        # is missing, update the expected generic location.
+        fs = port.unit_test_filesystem()
+        passing_run(['--no-show-results',
+                     'failures/unexpected/missing_text.html',
+                     'failures/unexpected/missing_image.html',
+                     'failures/unexpected/missing_audio.html'],
+                     tests_included=True, filesystem=fs)
+        file_list = fs.written_files.keys()
+        file_list.remove('/tmp/layout-test-results/tests_run0.txt')
+        self.assertEqual(len(file_list), 4)
+        self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"])
+        self.assertBaselines(file_list, "/failures/unexpected/missing_image", [".png"])
 
     def test_new_baseline(self):
@@ -679 +694 @@
         self.assertEqual(len(file_list), 4)
         self.assertBaselines(file_list,
-            "/platform/test-mac-leopard/passes/image")
+            "/platform/test-mac-leopard/passes/image", [".txt", ".png"])
         self.assertBaselines(file_list,
-            "/platform/test-mac-leopard/failures/expected/missing_image")
+            "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"])