Changeset 70277 in webkit

Timestamp:

Oct 21, 2010 5:04:39 PM (14 years ago)

Author:

mihaip@chromium.org

Message:

2010-10-21 Mihai Parparita <​mihaip@chromium.org>

Reviewed by Ojan Vafai.

Add support for --tolerance in NRWT
​https://bugs.webkit.org/show_bug.cgi?id=47959

Add support for the --tolerance flag in NRWT. The Port.diff_image
signature shouldn't need a tolerance parameter (it's not set per test),
just have ports that use it (currently only WebKitPort) read it from
the options object.

• Scripts/webkitpy/layout_tests/port/chromium.py:
• Scripts/webkitpy/layout_tests/port/port_testcase.py:
• Scripts/webkitpy/layout_tests/port/test.py:
• Scripts/webkitpy/layout_tests/port/webkit.py:
• Scripts/webkitpy/layout_tests/run_webkit_tests.py:
• Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:

Location:

trunk/WebKitTools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/chromium.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/port_testcase.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/test.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/webkit.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py (modified) (2 diffs)

trunk/WebKitTools/ChangeLog

@@ +1 @@
+2010-10-21  Mihai Parparita  <mihaip@chromium.org>
+
+        Reviewed by Ojan Vafai.
+
+        Add support for --tolerance in NRWT
+        https://bugs.webkit.org/show_bug.cgi?id=47959
+        
+        Add support for the --tolerance flag in NRWT. The Port.diff_image
+        signature shouldn't need a tolerance parameter (it's not set per test),
+        just have ports that use it (currently only WebKitPort) read it from 
+        the options object.
+
+        * Scripts/webkitpy/layout_tests/port/chromium.py:
+        * Scripts/webkitpy/layout_tests/port/port_testcase.py:
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        * Scripts/webkitpy/layout_tests/port/webkit.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
+
 2010-10-21  Eric Seidel  <eric@webkit.org>
 

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py

@@ -136 +136 @@
 
     def diff_image(self, expected_contents, actual_contents,
-                   diff_filename=None, tolerance=0):
+                   diff_filename=None):
         executable = self._path_to_image_diff()
 

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py

@@ -36 +36 @@
 mock_options = mocktool.MockOptions(results_directory='layout-test-results',
                                     use_apache=True,
-                                    configuration='Release')
+                                    configuration='Release',
+                                    tolerance=None)
 
 

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py

@@ -138 +138 @@
 
     def diff_image(self, expected_contents, actual_contents,
-                   diff_filename=None, tolerance=0):
+                   diff_filename=None):
         diffed = actual_contents != expected_contents
         if diffed and diff_filename:

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py

@@ -123 +123 @@
 
     def diff_image(self, expected_contents, actual_contents,
-                   diff_filename=None, tolerance=0.1):
+                   diff_filename=None):
         """Return True if the two files are different. Also write a delta
         image of the two images into |diff_filename| if it is not None."""
-
-        # FIXME: either expose the tolerance argument as a command-line
-        # parameter, or make it go away and always use exact matches.
 
         # Handle the case where the test didn't actually generate an image.
@@ -134 +131 @@
             return True
 
-        sp = self._diff_image_request(expected_contents, actual_contents,
-                                      tolerance)
+        sp = self._diff_image_request(expected_contents, actual_contents)
         return self._diff_image_reply(sp, diff_filename)
 
-    def _diff_image_request(self, expected_contents, actual_contents, tolerance):
+    def _diff_image_request(self, expected_contents, actual_contents):
+        # FIXME: use self.get_option('tolerance') and
+        # self.set_option_default('tolerance', 0.1) once that behaves correctly
+        # with default values.
+        if self._options.tolerance is not None:
+            tolerance = self._options.tolerance
+        else:
+            tolerance = 0.1
         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
         sp = server_process.ServerProcess(self, 'ImageDiff', command)

trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -1563 +1563 @@
         optparse.make_option("--no-pixel-tests", action="store_false",
             dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
-        # old-run-webkit-tests allows a specific tolerance: --tolerance t
-        # Ignore image differences less than this percentage (default: 0.1)
+        optparse.make_option("--tolerance",
+            help="Ignore image differences less than this percentage (some "
+                "ports may ignore this option)", type="float"),
         optparse.make_option("--results-directory",
             default="layout-test-results",

trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

@@ -49 +49 @@
 from webkitpy.layout_tests import run_webkit_tests
 from webkitpy.layout_tests.layout_package import dump_render_tree_thread
+from webkitpy.layout_tests.port.test import TestPort
 
 from webkitpy.thirdparty.mock import Mock
@@ -261 +262 @@
         self.assertEqual(user.url, '/tmp/foo/results.html')
 
+    def test_tolerance(self):
+        class ImageDiffTestPort(TestPort):
+            def diff_image(self, expected_contents, actual_contents,
+                   diff_filename=None):
+                self.tolerance_used_for_diff_image = self._options.tolerance
+                return True
+
+        def get_port_for_run(args):
+            options, parsed_args = run_webkit_tests.parse_args(args)
+            test_port = ImageDiffTestPort(options=options, user=MockUser())
+            passing_run(args=args, port_obj=test_port, tests_included=True)
+            return test_port
+
+        base_args = ['--pixel-tests', 'failures/expected/*']
+
+        # If we pass in an explicit tolerance argument, then that will be used.
+        test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
+        self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
+        test_port = get_port_for_run(base_args + ['--tolerance', '0'])
+        self.assertEqual(0, test_port.tolerance_used_for_diff_image)
+
+        # Otherwise the port's default tolerance behavior (including ignoring it)
+        # should be used.
+        test_port = get_port_for_run(base_args)
+        self.assertEqual(None, test_port.tolerance_used_for_diff_image)