⚠ Archived content — this site is no longer maintained.   Current WebKit documentation is at docs.webkit.org.

Changeset 99818 in webkit


Ignore:
Timestamp:
Nov 9, 2011, 11:39:50 PM (15 years ago)
Author:
hayato@chromium.org
Message:

[NRWT] Make results.html aware of reference filename in reftests.
​https://bugs.webkit.org/show_bug.cgi?id=71574

Reviewed by Ryosuke Niwa.

If a reference filename is different from the default one, include that in result summary
so results.html can link to the correct reference file.

Tools:

  • Scripts/webkitpy/layout_tests/controllers/manager.py:
  • Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:

LayoutTests:

  • fast/harness/resources/results-test.js:
  • fast/harness/results.html:
  • platform/chromium/fast/harness/results-expected.txt:
Location:
trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r99817 r99818  
     12011-11-09  Hayato Ito  <hayato@chromium.org>
     2
     3        [NRWT] Make results.html aware of reference filename in reftests.
     4        https://bugs.webkit.org/show_bug.cgi?id=71574
     5
     6        Reviewed by Ryosuke Niwa.
     7
     8        If a reference filename is different from the default one, include that in result summary
     9        so results.html can link to the correct reference file.
     10
     11        * fast/harness/resources/results-test.js:
     12        * fast/harness/results.html:
     13        * platform/chromium/fast/harness/results-expected.txt:
     14
    1152011-11-09  Andrew Scherkus  <scherkus@chromium.org>
    216

  • trunk/LayoutTests/fast/harness/resources/results-test.js

    r98031 r99818  
    264264    results.tests['bar-reftest-mismatch.html'].is_mismatch_reftest = true;
    265265    runSingleRowTest(results, false, '', 'ref mismatch html actual ');
     266
     267    results = mockResults();
     268    results.tests['bar-reftest.html'] = mockExpectation('PASS', 'IMAGE', 1);
     269    results.tests['bar-reftest.html'].is_reftest = true;
     270    results.tests['bar-reftest.html'].ref_file = 'common.html';
     271    runSingleRowTest(results, false, '', 'ref html images diff (1%) ');
     272    runTest(results, function() {
     273        assertTrue(document.getElementsByClassName('result-link')[0].getAttribute('href') == 'common.html');
     274    });
     275
     276    results = mockResults();
     277    results.tests['bar-reftest.html'] = mockExpectation('PASS', 'IMAGE');
     278    results.tests['bar-reftest.html'].is_mismatch_reftest = true;
     279    results.tests['bar-reftest.html'].ref_file = 'common.html';
     280    runSingleRowTest(results, false, '', 'ref mismatch html actual ');
     281    runTest(results, function() {
     282        assertTrue(document.getElementsByClassName('result-link')[0].getAttribute('href') == 'common.html');
     283    });
    266284
    267285    results = mockResults();

  • trunk/LayoutTests/fast/harness/results.html

    r98031 r99818  
    407407function resultLink(testPrefix, suffix, contents)
    408408{
    409     return '<a class=result-link href="' + testPrefix + suffix + '" data-prefix="' + testPrefix + '">' + contents + '</a> ';
     409    return referenceLink(testPrefix, testPrefix + suffix, contents);
     410}
     411
     412function referenceLink(testPrefix, reference_filename, contents)
     413{
     414    return '<a class=result-link href="' + reference_filename + '" data-prefix="' + testPrefix + '">' + contents + '</a> ';
    410415}
    411416
    … …  
    542547
    543548        if (testObject.is_mismatch_reftest) {
    544             row += resultLink(test_prefix, '-expected-mismatch.html', 'ref mismatch html') +
    545                 resultLink(test_prefix, '-actual.png', 'actual');
     549            if (testObject.ref_file)
     550                row += referenceLink(test_prefix, testObject.ref_file, 'ref mismatch html');
     551            else
     552                row += resultLink(test_prefix, '-expected-mismatch.html', 'ref mismatch html');
     553            row += resultLink(test_prefix, '-actual.png', 'actual');
    546554        } else {
    547             if (testObject.is_reftest)
    548                 row += resultLink(test_prefix, '-expected.html', 'ref html');
    549            
     555            if (testObject.is_reftest) {
     556                if (testObject.ref_file)
     557                    row += referenceLink(test_prefix, testObject.ref_file, 'ref html');
     558                else
     559                    row += resultLink(test_prefix, '-expected.html', 'ref html');
     560            }
    550561            if (globalState().shouldToggleImages) {
    551562                row += resultLink(test_prefix, '-diffs.html', 'images');

  • trunk/LayoutTests/platform/chromium/fast/harness/results-expected.txt

    r98031 r99818  
    9797TEST-16: PASS
    9898TEST-16: PASS
    99 TEST-17: PASS
    100 TEST-17: PASS
     99TEST-16: PASS
     100TEST-16: PASS
     101TEST-16: PASS
    101102TEST-17: PASS
    102103TEST-18: PASS
    … …  
    104105TEST-18: PASS
    105106TEST-18: PASS
     107TEST-18: PASS
     108TEST-18: PASS
    106109TEST-19: PASS
    107 TEST-19: PASS
    108 TEST-20: PASS
    109 TEST-20: PASS
    110110TEST-20: PASS
    111111TEST-20: PASS
    … …  
    113113TEST-21: PASS
    114114TEST-21: PASS
     115TEST-21: PASS
     116TEST-22: PASS
     117TEST-22: PASS
    115118TEST-22: PASS
    116119TEST-22: PASS
    117120TEST-23: PASS
     121TEST-23: PASS
     122TEST-24: PASS
     123TEST-24: PASS
     124TEST-24: PASS
    118125TEST-24: PASS
    119126TEST-24: PASS
    120127TEST-25: PASS
     128TEST-25: PASS
     129TEST-26: PASS
    121130TEST-26: PASS
    122131TEST-27: PASS
    123132TEST-28: PASS
    124133TEST-28: PASS
    125 TEST-28: PASS
    126 TEST-29: PASS
    127134TEST-29: PASS
    128135TEST-30: PASS
    129 TEST-30: PASS
    130 TEST-30: PASS
    131 TEST-30: PASS
    132 TEST-30: PASS
    133 TEST-30: PASS
    134 TEST-30: PASS
    135 TEST-30: PASS
    136 TEST-30: PASS
    137 TEST-30: PASS
    138 TEST-30: PASS
    139 TEST-30: PASS
    140 TEST-30: PASS
    141 TEST-30: PASS
    142 TEST-30: PASS
    143 TEST-30: PASS
    144136TEST-31: PASS
    145 TEST-31: PASS
    146 TEST-31: PASS
    147 TEST-31: PASS
    148 TEST-31: PASS
    149 TEST-31: PASS
    150 TEST-31: PASS
    151 TEST-31: PASS
    152 TEST-31: PASS
    153 TEST-31: PASS
    154 TEST-32: PASS
    155137TEST-32: PASS
    156138TEST-32: PASS
    … …  
    158140TEST-33: PASS
    159141TEST-33: PASS
    160 TEST-33: PASS
    161 TEST-33: PASS
    162 TEST-33: PASS
    163 TEST-33: PASS
    164 TEST-33: PASS
    165 TEST-33: PASS
     142TEST-34: PASS
     143TEST-34: PASS
     144TEST-34: PASS
     145TEST-34: PASS
     146TEST-34: PASS
     147TEST-34: PASS
     148TEST-34: PASS
     149TEST-34: PASS
     150TEST-34: PASS
     151TEST-34: PASS
     152TEST-34: PASS
     153TEST-34: PASS
     154TEST-34: PASS
     155TEST-34: PASS
     156TEST-34: PASS
     157TEST-34: PASS
     158TEST-35: PASS
     159TEST-35: PASS
     160TEST-35: PASS
     161TEST-35: PASS
     162TEST-35: PASS
     163TEST-35: PASS
     164TEST-35: PASS
     165TEST-35: PASS
     166TEST-35: PASS
     167TEST-35: PASS
     168TEST-36: PASS
     169TEST-36: PASS
     170TEST-36: PASS
     171TEST-36: PASS
     172TEST-37: PASS
     173TEST-37: PASS
     174TEST-37: PASS
     175TEST-37: PASS
     176TEST-37: PASS
     177TEST-37: PASS
     178TEST-37: PASS
     179TEST-37: PASS

  • trunk/Tools/ChangeLog

    r99811 r99818  
     12011-11-09  Hayato Ito  <hayato@chromium.org>
     2
     3        [NRWT] Make results.html aware of reference filename in reftests.
     4        https://bugs.webkit.org/show_bug.cgi?id=71574
     5
     6        Reviewed by Ryosuke Niwa.
     7
     8        If a reference filename is different from the default one, include that in result summary
     9        so results.html can link to the correct reference file.
     10
     11        * Scripts/webkitpy/layout_tests/controllers/manager.py:
     12        * Scripts/webkitpy/layout_tests/controllers/manager_unittest.py:
     13
    1142011-11-09  Kevin Ollivier  <kevino@theolliviers.com>
    215

  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

    r99461 r99818  
    6969
    7070
     71def interpret_test_failures(port, test_name, failures):
     72    """Interpret test failures and returns a test result as dict.
     73
     74    Args:
     75        port: interface to port-specific hooks
     76        test_name: test name relative to layout_tests directory
     77        failures: list of test failures
     78    Returns:
     79        A dictionary like {'is_reftest': True, ...}
     80    """
     81    test_dict = {}
     82    failure_types = [type(failure) for failure in failures]
     83    # FIXME: get rid of all this is_* values once there is a 1:1 map between
     84    # TestFailure type and test_expectations.EXPECTATION.
     85    if test_failures.FailureMissingAudio in failure_types:
     86        test_dict['is_missing_audio'] = True
     87
     88    for failure in failures:
     89        if isinstance(failure, test_failures.FailureImageHashMismatch):
     90            test_dict['image_diff_percent'] = failure.diff_percent
     91        elif isinstance(failure, test_failures.FailureReftestMismatch):
     92            test_dict['is_reftest'] = True
     93            if failure.reference_filename != port.reftest_expected_filename(test_name):
     94                test_dict['ref_file'] = port.relative_test_filename(failure.reference_filename)
     95        elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
     96            test_dict['is_mismatch_reftest'] = True
     97            if failure.reference_filename != port.reftest_expected_mismatch_filename(test_name):
     98                test_dict['ref_file'] = port.relative_test_filename(failure.reference_filename)
     99
     100    if test_failures.FailureMissingResult in failure_types:
     101        test_dict['is_missing_text'] = True
     102
     103    if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types:
     104        test_dict['is_missing_image'] = True
     105    return test_dict
     106
     107
    71108# FIXME: This should be on the Manager class (since that's the only caller)
    72109# or split off from Manager onto another helper class, but should not be a free function.
    … …  
    161198        # and only set it if there actually is stderr data.
    162199
    163         failure_types = [type(f) for f in result.failures]
    164         # FIXME: get rid of all this is_* values once there is a 1:1 map between
    165         # TestFailure type and test_expectations.EXPECTATION.
    166         if test_failures.FailureMissingAudio in failure_types:
    167             test_dict['is_missing_audio'] = True
    168 
    169         if test_failures.FailureReftestMismatch in failure_types:
    170             test_dict['is_reftest'] = True
    171 
    172         for f in result.failures:
    173             if 'is_reftest' in result.failures:
    174                 test_dict['is_reftest'] = True
    175             if type(f) is test_failures.FailureImageHashMismatch:
    176                 test_dict['image_diff_percent'] = f.diff_percent
    177 
    178         if test_failures.FailureReftestMismatchDidNotOccur in failure_types:
    179             test_dict['is_mismatch_reftest'] = True
    180 
    181         if test_failures.FailureMissingResult in failure_types:
    182             test_dict['is_missing_text'] = True
    183 
    184         if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types:
    185             test_dict['is_missing_image'] = True
     200        test_dict.update(interpret_test_failures(port_obj, test_name, result.failures))
    186201
    187202        # Store test hierarchically by directory. e.g.

  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py

    r99773 r99818  
    4343from webkitpy import layout_tests
    4444from webkitpy.layout_tests import run_webkit_tests
    45 from webkitpy.layout_tests.controllers.manager import Manager, natural_sort_key, test_key, TestRunInterruptedException, TestShard
     45from webkitpy.layout_tests.controllers.manager import interpret_test_failures,  Manager, natural_sort_key, test_key, TestRunInterruptedException, TestShard
     46from webkitpy.layout_tests.models import test_failures
    4647from webkitpy.layout_tests.models.result_summary import ResultSummary
    4748from webkitpy.layout_tests.views import printing
    … …  
    305306
    306307
     308class ResultSummaryTest(unittest.TestCase):
     309
     310    def setUp(self):
     311        host = MockHost()
     312        self.port = host.port_factory.get(port_name='test')
     313
     314    def test_interpret_test_failures(self):
     315        test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
     316            [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'))])
     317        self.assertTrue('is_reftest' in test_dict)
     318        self.assertFalse('is_mismatch_reftest' in test_dict)
     319        self.assertFalse('ref_file' in test_dict)
     320
     321        test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
     322            [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/common.html'))])
     323        self.assertTrue('is_reftest' in test_dict)
     324        self.assertFalse('is_mismatch_reftest' in test_dict)
     325        self.assertEqual(test_dict['ref_file'], 'foo/common.html')
     326
     327        test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
     328            [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
     329        self.assertFalse('is_reftest' in test_dict)
     330        self.assertTrue(test_dict['is_mismatch_reftest'])
     331        self.assertFalse('ref_file' in test_dict)
     332
     333        test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
     334            [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/common.html'))])
     335        self.assertFalse('is_reftest' in test_dict)
     336        self.assertTrue(test_dict['is_mismatch_reftest'])
     337        self.assertEqual(test_dict['ref_file'], 'foo/common.html')
     338
     339
    307340if __name__ == '__main__':
    308341    port_testcase.main()
Note: See TracChangeset for help on using the changeset viewer.