Changeset 96272 in webkit


Ignore:
Timestamp:
Sep 28, 2011 5:09:39 PM (13 years ago)
Author:
dpranke@chromium.org
Message:

Modify new-run-webkit-tests to return the percentage difference in image diffs where possible.

​https://bugs.webkit.org/show_bug.cgi?id=67253

Reviewed by Eric Seidel.

Tools:

Nearly all of the work in this was done by Simon Fraser; I'm
just repackaging it and cleaning it up a bit. This change
modifies port.diff_image() to return a tuple of (pass/fail,
%age), adds the value to the FailureImageHashMismatch, and
writes the value into the full_results.json files to be
displayed in the results page.

  • Scripts/webkitpy/layout_tests/controllers/manager.py:
  • Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py:
  • Scripts/webkitpy/layout_tests/models/test_failures.py:
  • Scripts/webkitpy/layout_tests/port/base.py:
  • Scripts/webkitpy/layout_tests/port/chromium.py:
  • Scripts/webkitpy/layout_tests/port/chromium_unittest.py:
  • Scripts/webkitpy/layout_tests/port/port_testcase.py:
  • Scripts/webkitpy/layout_tests/port/test.py:
  • Scripts/webkitpy/layout_tests/port/webkit.py:
  • Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

LayoutTests:

  • fast/harness/resources/results-test.js:

(mockExpectation):
(runDefaultSingleRowTest):
():

  • fast/harness/results.html:
Location:
trunk
Files:
16 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r96270 r96272  
     12011-09-28  Dirk Pranke  <dpranke@chromium.org>
     2
     3        Modify new-run-webkit-tests to return the percentage difference in image diffs where possible.
     4
     5        https://bugs.webkit.org/show_bug.cgi?id=67253
     6
     7        Reviewed by Eric Seidel.
     8
     9        * fast/harness/resources/results-test.js:
     10        (mockExpectation):
     11        (runDefaultSingleRowTest):
     12        ():
     13        * fast/harness/results.html:
     14
    1152011-09-28  Chris Rogers  <crogers@google.com>
    216

  • trunk/LayoutTests/fast/harness/resources/results-test.js

    r89873 r96272  
    3535}
    3636
    37 function mockExpectation(expected, actual)
    38 {
     37function mockExpectation(expected, actual, diff_percentage)
     38{
     39    diff_percentage = (typeof(diff_percentage) == undefined) ? 0 : diff_percentage;
    3940    return {
    4041        expected: expected,
    4142        time_ms: 1,
    4243        actual: actual,
     44        image_diff_percent: diff_percentage,
    4345        has_stderr: false
    4446    };
    … …  
    8486}
    8587
    86 function runDefaultSingleRowTest(test, expected, actual, isExpected, textResults, imageResults)
    87 {
    88     results = mockResults();
    89     results.tests[test] = mockExpectation(expected, actual);
     88function runDefaultSingleRowTest(test, expected, actual, diff_percentage, isExpected, textResults, imageResults)
     89{
     90    results = mockResults();
     91    results.tests[test] = mockExpectation(expected, actual, diff_percentage);
    9092    runSingleRowTest(results, isExpected, textResults, imageResults);
    9193}
    … …  
    246248    });
    247249
    248     runDefaultSingleRowTest('bar-skip.html', 'TEXT', 'SKIP', true, '', '');
    249     runDefaultSingleRowTest('bar-flaky-fail.html', 'PASS FAIL', 'TEXT', true, 'expected actual diff ', '');
    250     runDefaultSingleRowTest('bar-flaky-fail-unexpected.html', 'PASS TEXT', 'IMAGE', false, '', 'images diff ');
    251     runDefaultSingleRowTest('bar-audio.html', 'TEXT', 'AUDIO', false, 'expected audio actual audio ', '');
    252     runDefaultSingleRowTest('bar-image.html', 'TEXT', 'IMAGE', false, '', 'images diff ');
    253     runDefaultSingleRowTest('bar-image-plus-text.html', 'TEXT', 'IMAGE+TEXT', false, 'expected actual diff ', 'images diff ');
    254 
    255     results = mockResults();
    256     results.tests['bar-reftest.html'] = mockExpectation('PASS', 'IMAGE');
     250    runDefaultSingleRowTest('bar-skip.html', 'TEXT', 'SKIP', 0, true, '', '');
     251    runDefaultSingleRowTest('bar-flaky-fail.html', 'PASS FAIL', 'TEXT', 0, true, 'expected actual diff ', '');
     252    runDefaultSingleRowTest('bar-flaky-fail-unexpected.html', 'PASS TEXT', 'IMAGE', 1, false, '', 'images diff (1%) ');
     253    runDefaultSingleRowTest('bar-audio.html', 'TEXT', 'AUDIO', 0, false, 'expected audio actual audio ', '');
     254    runDefaultSingleRowTest('bar-image.html', 'TEXT', 'IMAGE', 1, false, '', 'images diff (1%) ');
     255    runDefaultSingleRowTest('bar-image-plus-text.html', 'TEXT', 'IMAGE+TEXT', 1, false, 'expected actual diff ', 'images diff (1%) ');
     256
     257    results = mockResults();
     258    results.tests['bar-reftest.html'] = mockExpectation('PASS', 'IMAGE', 1);
    257259    results.tests['bar-reftest.html'].is_reftest = true;
    258     runSingleRowTest(results, false, '', 'ref html images diff ');
     260    runSingleRowTest(results, false, '', 'ref html images diff (1%) ');
    259261
    260262    results = mockResults();
    … …  
    411413
    412414    results = mockResults();
    413     results.tests['bar.html'] = mockExpectation('PASS', 'IMAGE');
    414     runTest(results, function() {
    415         assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == 'images diff ');
     415    results.tests['bar.html'] = mockExpectation('PASS', 'IMAGE', 1);
     416    runTest(results, function() {
     417        assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == 'images diff (1%) ');
    416418
    417419        document.getElementById('toggle-images').checked = false;
    … …  
    419421        updateTogglingImages();
    420422        // FIXME: We get extra spaces in the DOM every time we enable/disable image toggling.
    421         assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == 'expected actual  diff ');
     423        assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == 'expected actual  diff (1%) ');
    422424       
    423425        document.getElementById('toggle-images').checked = true;
    424426        updateTogglingImages();
    425         assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == ' images   diff ');
    426     });
    427    
    428     results = mockResults();
    429     results.tests['reading-options-from-localstorage.html'] = mockExpectation('IMAGE+TEXT', 'IMAGE+TEXT');
     427        assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == ' images   diff (1%) ');
     428    });
     429   
     430    results = mockResults();
     431    results.tests['reading-options-from-localstorage.html'] = mockExpectation('IMAGE+TEXT', 'IMAGE+TEXT', 1);
    430432    runTest(results, function() {
    431433        assertTrue(window.getComputedStyle(document.querySelector('tbody'), null)['display'] != 'none');
    432         assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == 'expected actual  diff ');
     434        assertTrue(document.querySelector('tbody td:nth-child(3)').textContent == 'expected actual  diff (1%) ');
    433435    }, '{"toggle-images":false,"unexpected-results":false}');
    434436

  • trunk/LayoutTests/fast/harness/results.html

    r94212 r96272  
    546546            }
    547547
    548             // FIXME: old-run-webkit-tests shows the diff percentage as the text contents of the "diff" link.
    549             row += resultLink(test_prefix, '-diff.png', 'diff');
     548            var diff = testObject.image_diff_percent;
     549            row += resultLink(test_prefix, '-diff.png', 'diff (' + diff + '%)');
    550550        }
    551551    }

  • trunk/Tools/ChangeLog

    r96268 r96272  
     12011-09-28  Dirk Pranke  <dpranke@chromium.org>
     2
     3        Modify new-run-webkit-tests to return the percentage difference in image diffs where possible.
     4
     5        https://bugs.webkit.org/show_bug.cgi?id=67253
     6
     7        Reviewed by Eric Seidel.
     8
     9        Nearly all of the work in this was done by Simon Fraser; I'm
     10        just repackaging it and cleaning it up a bit. This change
     11        modifies port.diff_image() to return a tuple of (pass/fail,
     12        %age), adds the value to the FailureImageHashMismatch, and
     13        writes the value into the full_results.json files to be
     14        displayed in the results page.
     15
     16        * Scripts/webkitpy/layout_tests/controllers/manager.py:
     17        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
     18        * Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py:
     19        * Scripts/webkitpy/layout_tests/models/test_failures.py:
     20        * Scripts/webkitpy/layout_tests/port/base.py:
     21        * Scripts/webkitpy/layout_tests/port/chromium.py:
     22        * Scripts/webkitpy/layout_tests/port/chromium_unittest.py:
     23        * Scripts/webkitpy/layout_tests/port/port_testcase.py:
     24        * Scripts/webkitpy/layout_tests/port/test.py:
     25        * Scripts/webkitpy/layout_tests/port/webkit.py:
     26        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
     27
    1282011-09-25  Kentaro Hara  <haraken@chromium.org>
    229

  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

    r95441 r96272  
    171171            if 'is_reftest' in result.failures:
    172172                test_dict['is_reftest'] = True
     173            if type(f) is test_failures.FailureImageHashMismatch:
     174                test_dict['image_diff_percent'] = f.diff_percent
    173175
    174176        if test_failures.FailureReftestMismatchDidNotOccur in failure_types:

  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

    r90532 r96272  
    277277            failures.append(test_failures.FailureMissingImageHash())
    278278        elif driver_output.image_hash != expected_driver_output.image_hash:
    279             driver_output.image_diff = self._port.diff_image(driver_output.image, expected_driver_output.image)
     279            diff_result = self._port.diff_image(driver_output.image, expected_driver_output.image)
     280            driver_output.image_diff = diff_result[0]
    280281            if driver_output.image_diff:
    281                 failures.append(test_failures.FailureImageHashMismatch())
     282                failures.append(test_failures.FailureImageHashMismatch(diff_result[1]))
    282283        return failures
    283284

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py

    r90532 r96272  
    7171            writer.write_image_files(driver_output.image, expected_driver_output.image)
    7272            # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
    73             image_diff = port.diff_image(driver_output.image, expected_driver_output.image)
     73            image_diff = port.diff_image(driver_output.image, expected_driver_output.image)[0]
    7474            if image_diff:
    7575                writer.write_image_diff_files(image_diff)

  • trunk/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py

    r90532 r96272  
    168168class FailureImageHashMismatch(TestFailure):
    169169    """Image hashes didn't match."""
     170    def __init__(self, diff_percent=0):
     171        self.diff_percent = diff_percent
    170172
    171173    @staticmethod

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py

    r93166 r96272  
    258258
    259259    def diff_image(self, expected_contents, actual_contents, tolerance=0):
    260         """Compare two images and return an image diff.
     260        """Compare two images and return a tuple of an image diff, and a percentage difference (0-100).
    261261
    262262        |tolerance| should be a percentage value (0.0 - 100.0).

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/chromium.py

    r95875 r96272  
    169169        # If only one of them exists, return that one.
    170170        if not actual_contents and not expected_contents:
    171             return None
     171            return (None, 0)
    172172        if not actual_contents:
    173             return expected_contents
     173            return (expected_contents, 0)
    174174        if not expected_contents:
    175             return actual_contents
     175            return (actual_contents, 0)
    176176
    177177        tempdir = self._filesystem.mkdtemp()
    … …  
    214214                result = self._filesystem.read_binary_file(native_diff_filename)
    215215            self._filesystem.rmtree(str(tempdir))
    216         return result
     216        return (result, 0)  # FIXME: how to get % diff?
    217217
    218218    def path_from_chromium_base(self, *comps):

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py

    r94219 r96272  
    263263        # Images are different.
    264264        port._executive = executive_mock.MockExecutive2(run_command_fn=mock_run_command)
    265         self.assertEquals(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL"))
     265        self.assertEquals(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0])
    266266
    267267        # Images are the same.
    268268        port._executive = executive_mock.MockExecutive2(exit_code=0)
    269         self.assertEquals(None, port.diff_image("EXPECTED", "ACTUAL"))
     269        self.assertEquals(None, port.diff_image("EXPECTED", "ACTUAL")[0])
    270270
    271271        # There was some error running image_diff.

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py

    r93176 r96272  
    227227        tmpfd.close()
    228228
    229         self.assertFalse(port.diff_image(contents1, contents1))
    230         self.assertTrue(port.diff_image(contents1, contents2))
    231 
    232         self.assertTrue(port.diff_image(contents1, contents2, tmpfile))
     229        self.assertFalse(port.diff_image(contents1, contents1)[0])
     230        self.assertTrue(port.diff_image(contents1, contents2)[0])
     231
     232        self.assertTrue(port.diff_image(contents1, contents2, tmpfile)[0])
    233233
    234234        port._filesystem.remove(tmpfile)
    … …  
    238238        if not port:
    239239            return
    240         self.assertFalse(port.diff_image(None, None))
    241         self.assertFalse(port.diff_image(None, ''))
    242         self.assertFalse(port.diff_image('', None))
    243         self.assertFalse(port.diff_image('', ''))
     240        self.assertFalse(port.diff_image(None, None)[0])
     241        self.assertFalse(port.diff_image(None, '')[0])
     242        self.assertFalse(port.diff_image('', None)[0])
     243        self.assertFalse(port.diff_image('', '')[0])
    244244
    245245    def test_diff_image__missing_actual(self):
    … …  
    247247        if not port:
    248248            return
    249         self.assertTrue(port.diff_image(None, 'foo'))
    250         self.assertTrue(port.diff_image('', 'foo'))
     249        self.assertTrue(port.diff_image(None, 'foo')[0])
     250        self.assertTrue(port.diff_image('', 'foo')[0])
    251251
    252252    def test_diff_image__missing_expected(self):
    … …  
    254254        if not port:
    255255            return
    256         self.assertTrue(port.diff_image('foo', None))
    257         self.assertTrue(port.diff_image('foo', ''))
     256        self.assertTrue(port.diff_image('foo', None)[0])
     257        self.assertTrue(port.diff_image('foo', '')[0])
    258258
    259259    def test_check_build(self):

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/test.py

    r94316 r96272  
    326326        diffed = actual_contents != expected_contents
    327327        if diffed:
    328             return "< %s\n---\n> %s\n" % (expected_contents, actual_contents)
    329         return None
     328            return ["< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1]
     329        return (None, 0)
    330330
    331331    def layout_tests_dir(self):

  • trunk/Tools/Scripts/webkitpy/layout_tests/port/webkit.py

    r95911 r96272  
    152152        # FIXME: need unit tests for this.
    153153        if not actual_contents and not expected_contents:
    154             return None
     154            return (None, 0)
    155155        if not actual_contents or not expected_contents:
    156156            # FIXME: It's not clear what we should return in this case.
    157157            # Maybe we should throw an exception?
    158             return True
     158            return (True, 0)
    159159
    160160        process = self._start_image_diff_process(expected_contents, actual_contents)
    … …  
    181181        deadline = time.time() + timeout
    182182        output = sp.read_line(timeout)
     183        output_image = ""
     184        diff_percent = 0
    183185        while not sp.timed_out and not sp.crashed and output:
    184186            if output.startswith('Content-Length'):
    … …  
    186188                content_length = int(m.group(1))
    187189                timeout = deadline - time.time()
    188                 output = sp.read(timeout, content_length)
     190                output_image = sp.read(timeout, content_length)
     191                output = sp.read_line(timeout)
    189192                break
    190193            elif output.startswith('diff'):
    … …  
    202205            m = re.match('diff: (.+)% (passed|failed)', output)
    203206            if m.group(2) == 'passed':
    204                 return None
    205         return output
     207                return [None, 0]
     208            diff_percent = float(m.group(1))
     209        return (output_image, diff_percent)
    206210
    207211    def default_results_directory(self):

  • trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

    r95441 r96272  
    5050    multiprocessing = None
    5151
     52try:
     53    import json
     54except ImportError:
     55    # python 2.5 compatibility
     56    import webkitpy.thirdparty.simplejson as json
     57
    5258# FIXME: remove this when we fix test-webkitpy to work properly on cygwin
    5359# (bug 63846).
    … …  
    581587            def diff_image(self, expected_contents, actual_contents):
    582588                self.tolerance_used_for_diff_image = self._options.tolerance
    583                 return True
     589                return (True, 1)
    584590
    585591        def get_port_for_run(args):
    … …  
    675681MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
    676682
     683
     684class EndToEndTest(unittest.TestCase):
     685    def parse_full_results(self, full_results_text):
     686        json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
     687        compressed_results = json.loads(json_to_eval)
     688        return compressed_results
     689
     690    def test_end_to_end(self):
     691        fs = unit_test_filesystem()
     692        res, out, err, user = logging_run(record_results=True, tests_included=True, filesystem=fs)
     693
     694        # Six tests should fail, so the return code should be 6.
     695        self.assertEquals(res, 6)
     696        results = self.parse_full_results(fs.files['/tmp/layout-test-results/full_results.json'])
     697
     698        # Check to ensure we're passing back image diff %age correctly.
     699        self.assertEquals(results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1)
     700
     701        # Check that we attempted to display the results page in a browser.
     702        self.assertTrue(user.opened_urls)
    677703
    678704class RebaselineTest(unittest.TestCase):

  • trunk/Tools/Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py

    r94031 r96272  
    515515
    516516        if is_image:
    517             return self._port.diff_image(output1, output2)
     517            return self._port.diff_image(output1, output2)[0]
    518518
    519519        return self._port.compare_text(output1, output2)
    … …  
    600600            old_output = self._filesystem.read_binary_file(old_file)
    601601            new_output = self._filesystem.read_binary_file(new_file)
    602             image_diff = self._port.diff_image(old_output, new_output)
     602            image_diff = self._port.diff_image(old_output, new_output)[0]
    603603            self._filesystem.write_binary_file(diff_file, image_diff)
    604604
Note: See TracChangeset for help on using the changeset viewer.