Changeset 287790 in webkit

Timestamp:

Jan 7, 2022, 3:19:31 PM (5 years ago)

Author:

timothy_horton@apple.com

Message:

A/B comparison tests fail trying to look up fuzzy match tolerance
​https://bugs.webkit.org/show_bug.cgi?id=234974

Reviewed by Aditya Keerthi.

• Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:

(SingleTestRunner._run_self_comparison_test):
(SingleTestRunner._compare_output_with_reference):
Disable fuzzy matching for A/B tests, because A) we don't really want it
and B) it crashes run-webkit-tests because there is no actual reference file
in which to find the tolerance value.

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/controllers/single_test_runner.py (modified) (4 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2022-01-07  Tim Horton  <timothy_horton@apple.com>
+
+        A/B comparison tests fail trying to look up fuzzy match tolerance
+        https://bugs.webkit.org/show_bug.cgi?id=234974
+
+        Reviewed by Aditya Keerthi.
+
+        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+        (SingleTestRunner._run_self_comparison_test):
+        (SingleTestRunner._compare_output_with_reference):
+        Disable fuzzy matching for A/B tests, because A) we don't really want it
+        and B) it crashes run-webkit-tests because there is no actual reference file
+        in which to find the tolerance value.
+
 2022-01-07  David Kilzer  <ddkilzer@apple.com>
 

trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

@@ -350 +350 @@
 
         test_full_path = self._port.abspath_for_test(self._test_name)
-        test_result = self._compare_output_with_reference(reference_output, test_output, test_full_path, False)
+        test_result = self._compare_output_with_reference(reference_output, test_output, test_full_path, False, allow_fuzzy_tolerance=False)
 
         assert(reference_output)
@@ -396 +396 @@
         return actualMaxDifference >= maxDifferenceMin and actualMaxDifference <= maxDifferenceMax and actualTotalPixels >= totalPixelsMin and actualTotalPixels <= totalPixelsMax
 
-    def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch):
+    def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch, allow_fuzzy_tolerance=True):
         total_test_time = reference_driver_output.test_time + actual_driver_output.test_time
         has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr()
@@ -412 +412 @@
             failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename))
         elif mismatch:
-            fuzzy_tolerance = self._fuzzy_tolerance_for_reference(reference_filename)
+            fuzzy_tolerance = None
+            if allow_fuzzy_tolerance:
+                fuzzy_tolerance = self._fuzzy_tolerance_for_reference(reference_filename)
             if fuzzy_tolerance:
                 diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
@@ -429 +431 @@
             diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image, tolerance=0)
 
-            fuzzy_tolerance = self._fuzzy_tolerance_for_reference(reference_filename)
+            fuzzy_tolerance = None
+            if allow_fuzzy_tolerance:
+                fuzzy_tolerance = self._fuzzy_tolerance_for_reference(reference_filename)
             if fuzzy_tolerance:
                 diff_result.passed = self._test_passes_fuzzy_matching(fuzzy_tolerance, diff_result.fuzzy_data)