Changeset 84294 in webkit


Ignore:
Timestamp:
Apr 19, 2011 2:18:52 PM (13 years ago)
Author:
ojan@chromium.org
Message:

2011-04-18 Ojan Vafai <​ojan@chromium.org>

Reviewed by Eric Seidel.

switch new-run-webkit-tests to using the new results file
​https://bugs.webkit.org/show_bug.cgi?id=58861

  • Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_runner.py:
Location:
trunk/Tools
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r84290 r84294  
     12011-04-18  Ojan Vafai  <ojan@chromium.org>
     2
     3        Reviewed by Eric Seidel.
     4
     5        switch new-run-webkit-tests to using the new results file
     6        https://bugs.webkit.org/show_bug.cgi?id=58861
     7
     8        * Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
     9        * Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py:
     10        * Scripts/webkitpy/layout_tests/layout_package/test_runner.py:
     11
    1122011-04-19  Renata Hodovan  <reni@webkit.org>
    213

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results.html

    r84267 r84294  
    301301    row += '<td>' + expected + '</td>';
    302302
    303   var isExpected = results.uses_expectations_file && expected == actual;
     303  var isExpected = actual == 'SKIP' || (results.uses_expectations_file && expected == actual);
    304304  html += '<tbody class="' + (isExpected ? 'expected' : '') + '"><tr>' + row + '</tr></tbody>';
    305305}
    … …  
    521521function updateExpectedResults()
    522522{
    523     if (document.querySelector('.unexpected-results').checked)
     523    var checkBox = document.querySelector('.unexpected-results');
     524    if (!checkBox || checkBox.checked)
    524525        unexpectedStyleNode.innerText = '.expected { display: none; }';
    525526    else

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py

    r83635 r84294  
    195195        fs = self._port._filesystem
    196196        if driver_output.timeout:
    197             failures.append(test_failures.FailureTimeout(reference_filename))
     197            failures.append(test_failures.FailureTimeout(bool(reference_filename)))
    198198
    199199        if reference_filename:
    … …  
    203203
    204204        if driver_output.crash:
    205             failures.append(test_failures.FailureCrash(reference_filename))
     205            failures.append(test_failures.FailureCrash(bool(reference_filename)))
    206206            _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, testname,
    207207                                                      driver_output.error))

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py

    r83475 r84294  
    104104        return cPickle.dumps(self)
    105105
    106     def result_html_output(self, filename):
    107         """Returns an HTML string to be included on the results.html page."""
    108         raise NotImplementedError
    109 
    110106    def should_kill_dump_render_tree(self):
    111107        """Returns True if we should kill DumpRenderTree before the next
    … …  
    113109        return False
    114110
    115     def relative_output_filename(self, filename, modifier):
    116         """Returns a relative filename inside the output dir that contains
    117         modifier.
    118 
    119         For example, if filename is fast\dom\foo.html and modifier is
    120         "-expected.txt", the return value is fast\dom\foo-expected.txt
    121 
    122         Args:
    123           filename: relative filename to test file
    124           modifier: a string to replace the extension of filename with
    125 
    126         Return:
    127           The relative windows path to the output filename
    128         """
    129         # FIXME: technically this breaks if files don't use ".ext" to indicate
    130         # the extension, but passing in a Filesystem object here is a huge
    131         # hassle.
    132         return filename[:filename.rfind('.')] + modifier
    133 
    134 
    135 class ComparisonTestFailure(TestFailure):
    136     """Base class that produces standard HTML output based on the result of the comparison test.
    137 
    138     Subclasses may commonly choose to override the ResultHtmlOutput, but still
    139     use the standard OutputLinks.
    140     """
    141 
    142     # Filename suffixes used by ResultHtmlOutput.
    143     OUT_FILENAMES = ()
    144 
    145     def output_links(self, filename, out_names):
    146         """Returns a string holding all applicable output file links.
    147 
    148         Args:
    149           filename: the test filename, used to construct the result file names
    150           out_names: list of filename suffixes for the files. If three or more
    151               suffixes are in the list, they should be [actual, expected, diff,
    152               wdiff]. Two suffixes should be [actual, expected], and a
    153               single item is the [actual] filename suffix.
    154               If out_names is empty, returns the empty string.
    155         """
    156         # FIXME: Seems like a bad idea to separate the display name data
    157         # from the path data by hard-coding the display name here
    158         # and passing in the path information via out_names.
    159         #
    160         # FIXME: Also, we don't know for sure that these files exist,
    161         # and we shouldn't be creating links to files that don't exist
    162         # (for example, if we don't actually have wdiff output).
    163         links = ['']
    164         uris = [self.relative_output_filename(filename, fn) for
    165                 fn in out_names]
    166         if len(uris) > 1:
    167             links.append("<a href='%s'>expected</a>" % uris[1])
    168         if len(uris) > 0:
    169             links.append("<a href='%s'>actual</a>" % uris[0])
    170         if len(uris) > 2:
    171             links.append("<a href='%s'>diff</a>" % uris[2])
    172         if len(uris) > 3:
    173             links.append("<a href='%s'>wdiff</a>" % uris[3])
    174         if len(uris) > 4:
    175             links.append("<a href='%s'>pretty diff</a>" % uris[4])
    176         return ' '.join(links)
    177 
    178     def result_html_output(self, filename):
    179         return self.message() + self.output_links(filename, self.OUT_FILENAMES)
    180 
    181111
    182112class FailureTimeout(TestFailure):
    183113    """Test timed out.  We also want to restart DumpRenderTree if this
    184114    happens."""
    185 
    186     def __init__(self, reference_filename=None):
    187         self.reference_filename = reference_filename
     115    def __init__(self, is_reftest=False):
     116        self.is_reftest = is_reftest
    188117
    189118    @staticmethod
    190119    def message():
    191120        return "Test timed out"
    192 
    193     def result_html_output(self, filename):
    194         if self.reference_filename:
    195             return "<strong>%s</strong> (occured in <a href=%s>expected html</a>)" % (
    196                 self.message(), self.reference_filename)
    197         return "<strong>%s</strong>" % self.message()
    198121
    199122    def should_kill_dump_render_tree(self):
    … …  
    203126class FailureCrash(TestFailure):
    204127    """DumpRenderTree crashed."""
    205 
    206     def __init__(self, reference_filename=None):
    207         self.reference_filename = reference_filename
     128    def __init__(self, is_reftest=False):
     129        self.is_reftest = is_reftest
    208130
    209131    @staticmethod
    210132    def message():
    211133        return "DumpRenderTree crashed"
    212 
    213     def result_html_output(self, filename):
    214         # FIXME: create a link to the minidump file
    215         stack = self.relative_output_filename(filename, "-stack.txt")
    216         if self.reference_filename:
    217             return "<strong>%s</strong> <a href=%s>stack</a> (occured in <a href=%s>expected html</a>)" % (
    218                 self.message(), stack, self.reference_filename)
    219         else:
    220             return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(), stack)
    221134
    222135    def should_kill_dump_render_tree(self):
    … …  
    224137
    225138
    226 class FailureMissingResult(ComparisonTestFailure):
     139class FailureMissingResult(TestFailure):
    227140    """Expected result was missing."""
    228     OUT_FILENAMES = ("-actual.txt",)
    229141
    230142    @staticmethod
    … …  
    232144        return "No expected results found"
    233145
    234     def result_html_output(self, filename):
    235         return ("<strong>%s</strong>" % self.message() +
    236                 self.output_links(filename, self.OUT_FILENAMES))
    237 
    238 
    239 class FailureTextMismatch(ComparisonTestFailure):
     146
     147class FailureTextMismatch(TestFailure):
    240148    """Text diff output failed."""
    241     # Filename suffixes used by ResultHtmlOutput.
    242     # FIXME: Why don't we use the constants from TestTypeBase here?
    243     OUT_FILENAMES = ("-actual.txt", "-expected.txt", "-diff.txt",
    244                      "-wdiff.html", "-pretty-diff.html")
    245149
    246150    @staticmethod
    … …  
    249153
    250154
    251 class FailureMissingImageHash(ComparisonTestFailure):
     155class FailureMissingImageHash(TestFailure):
    252156    """Actual result hash was missing."""
    253157    # Chrome doesn't know to display a .checksum file as text, so don't bother
    … …  
    258162        return "No expected image hash found"
    259163
    260     def result_html_output(self, filename):
    261         return "<strong>%s</strong>" % self.message()
    262 
    263 
    264 class FailureMissingImage(ComparisonTestFailure):
     164
     165class FailureMissingImage(TestFailure):
    265166    """Actual result image was missing."""
    266     OUT_FILENAMES = ("-actual.png",)
    267167
    268168    @staticmethod
    … …  
    270170        return "No expected image found"
    271171
    272     def result_html_output(self, filename):
    273         return ("<strong>%s</strong>" % self.message() +
    274                 self.output_links(filename, self.OUT_FILENAMES))
    275 
    276 
    277 class FailureImageHashMismatch(ComparisonTestFailure):
     172
     173class FailureImageHashMismatch(TestFailure):
    278174    """Image hashes didn't match."""
    279     OUT_FILENAMES = ("-actual.png", "-expected.png", "-diff.png")
    280175
    281176    @staticmethod
    … …  
    286181
    287182
    288 class FailureImageHashIncorrect(ComparisonTestFailure):
     183class FailureImageHashIncorrect(TestFailure):
    289184    """Actual result hash is incorrect."""
    290185    # Chrome doesn't know to display a .checksum file as text, so don't bother
    … …  
    295190        return "Images match, expected image hash incorrect. "
    296191
    297     def result_html_output(self, filename):
    298         return "<strong>%s</strong>" % self.message()
    299 
    300 
    301 class FailureReftestMismatch(ComparisonTestFailure):
     192
     193class FailureReftestMismatch(TestFailure):
    302194    """The result didn't match the reference rendering."""
    303195
    304     OUT_FILENAMES = ("-expected.html", "-expected.png", "-actual.png",
    305                      "-diff.png",)
    306 
    307196    @staticmethod
    308197    def message():
    309198        return "Mismatch with reference"
    310199
    311     def output_links(self, filename, out_names):
    312         links = ['']
    313         uris = [self.relative_output_filename(filename, output_filename)
    314                 for output_filename in out_names]
    315         for text, uri in zip(['-expected.html', 'expected', 'actual', 'diff'], uris):
    316             links.append("<a href='%s'>%s</a>" % (uri, text))
    317         return ' '.join(links)
    318 
    319 
    320 class FailureReftestMismatchDidNotOccur(ComparisonTestFailure):
     200
     201class FailureReftestMismatchDidNotOccur(TestFailure):
    321202    """Unexpected match between the result and the reference rendering."""
    322203
    323     OUT_FILENAMES = ("-expected-mismatch.html", "-actual.png",)
    324 
    325204    @staticmethod
    326205    def message():
    327206        return "Mismatch with the reference did not occur"
    328207
    329     def output_links(self, filename, out_names):
    330         links = ['']
    331         uris = [self.relative_output_filename(filename, output_filename)
    332                 for output_filename in out_names]
    333         for text, uri in zip(['-expected-mismatch.html', 'image'], uris):
    334             links.append("<a href='%s'>%s</a>" % (uri, text))
    335         return ' '.join(links)
    336 
    337 
    338 class FailureMissingAudio(ComparisonTestFailure):
     208
     209class FailureMissingAudio(TestFailure):
    339210    """Actual result image was missing."""
    340     OUT_FILENAMES = ("-actual.wav",)
    341211
    342212    @staticmethod
    … …  
    344214        return "No expected audio found"
    345215
    346     def result_html_output(self, filename):
    347         return ("<strong>%s</strong>" % self.message() +
    348                 self.output_links(filename, self.OUT_FILENAMES))
    349 
    350 
    351 class FailureAudioMismatch(ComparisonTestFailure):
     216
     217class FailureAudioMismatch(TestFailure):
    352218    """Audio files didn't match."""
    353     OUT_FILENAMES = ("-actual.wav", "-expected.wav")
    354219
    355220    @staticmethod

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py

    r76311 r84294  
    3535
    3636class Test(unittest.TestCase):
    37     def assertResultHtml(self, failure_obj):
    38         self.assertNotEqual(failure_obj.result_html_output('foo'), None)
    39 
    4037    def assert_loads(self, cls):
    4138        failure_obj = cls()
    … …  
    7471        self.assertRaises(ValueError, determine_result_type, [failure_obj])
    7572        self.assertRaises(NotImplementedError, failure_obj.message)
    76         self.assertRaises(NotImplementedError, failure_obj.result_html_output,
    77                           "foo.txt")
    7873
    7974    def test_loads(self):
    … …  
    9085        self.assertEqual(len(crash_set), 2)
    9186
    92     def test_relative_output_filename(self):
    93         # This could be any Failure* object, since we're testing a method
    94         # on the base class.
    95         failure_obj = FailureTextMismatch()
    96         actual_filename = failure_obj.relative_output_filename("fast/html/article-element.html", "-actual.txt")
    97         self.assertEquals(actual_filename, "fast/html/article-element-actual.txt")
    98 
    9987if __name__ == '__main__':
    10088    unittest.main()

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py

    r84200 r84294  
    7272            writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
    7373        elif isinstance(failure, test_failures.FailureCrash):
    74             if failure.reference_filename:
     74            if failure.is_reftest:
    7575                writer.write_crash_report(expected_driver_output.error)
    7676            else:

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py

    r84269 r84294  
    155155            tests[test]['is_reftest'] = True
    156156
     157        for f in result.failures:
     158            if 'is_reftest' in result.failures:
     159                tests[test]['is_reftest'] = True
     160
    157161        if test_failures.FailureReftestMismatchDidNotOccur in failure_types:
    158162            tests[test]['is_mismatch_reftest'] = True
    … …  
    693697        # Write the summary to disk (results.html) and display it if requested.
    694698        if not self._options.dry_run:
    695             wrote_results = self._write_results_html_file(result_summary)
    696             if self._options.show_results and wrote_results:
    697                 self._show_results_html_file()
    698             self._write_unexpected_results_html_file(unexpected_results)
     699            self._copy_results_html_file()
     700            if self._options.show_results:
     701                self._show_results_html_file(result_summary)
    699702
    700703        # Now that we've completed all the processing we can, we re-raise
    … …  
    11111114                    (len(results), desc[len(results) != 1], pct))
    11121115
    1113     def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
    1114         """
    1115         test_files = a list of file paths
    1116         failures = dictionary mapping test paths to failure objects
    1117         title = title printed at top of test
    1118         override_time = current time (used by unit tests)
    1119         """
    1120         page = """<html>
    1121   <head>
    1122     <title>Layout Test Results (%(time)s)</title>
    1123   </head>
    1124   <body>
    1125     <h2>%(title)s (%(time)s)</h2>
    1126         """ % {'title': title, 'time': override_time or time.asctime()}
    1127 
    1128         for test_file in sorted(test_files):
    1129             test_name = self._port.relative_test_filename(test_file)
    1130             test_url = self._port.filename_to_uri(test_file)
    1131             page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
    1132             test_failures = failures.get(test_file, [])
    1133             for failure in test_failures:
    1134                 page += (u"&nbsp;&nbsp;%s<br/>" %
    1135                          failure.result_html_output(test_name))
    1136             page += "</p>\n"
    1137         page += "</body></html>\n"
    1138         return page
    1139 
    1140     def _write_results_html_file(self, result_summary):
    1141         """Write results.html which is a summary of tests that failed.
    1142 
    1143         Args:
    1144           result_summary: a summary of the results :)
    1145 
    1146         Returns:
    1147           True if any results were written (since expected failures may be
    1148           omitted)
    1149         """
    1150         # test failures
    1151         if self._options.full_results_html:
    1152             results_title = "Test Failures"
    1153             test_files = result_summary.failures.keys()
    1154         else:
    1155             results_title = "Unexpected Test Failures"
    1156             unexpected_failures = self._get_failures(result_summary,
    1157                 include_crashes=True)
    1158             test_files = unexpected_failures.keys()
    1159         if not len(test_files):
    1160             return False
    1161 
    1162         out_filename = self._fs.join(self._results_directory, "results.html")
    1163         with self._fs.open_text_file_for_writing(out_filename) as results_file:
    1164             html = self._results_html(test_files, result_summary.failures, results_title)
    1165             results_file.write(html)
    1166 
    1167         return True
    1168 
    1169     # FIXME: Have this replace results.html and have a checkbox for whether to show expected failures or not.
    1170     def _write_unexpected_results_html_file(self, unexpected_results):
     1116    def _copy_results_html_file(self):
    11711117        base_dir = self._port.path_from_webkit_base('Tools', 'Scripts', 'webkitpy', 'layout_tests', 'layout_package')
    11721118        results_file = self._fs.join(base_dir, 'json_results.html')
    11731119        # FIXME: What should we do if this doesn't exist (e.g., in unit tests)?
    11741120        if self._fs.exists(results_file):
    1175             self._fs.copyfile(results_file, self._fs.join(self._results_directory, "json_results.html"))
    1176 
    1177     def _show_results_html_file(self):
     1121            self._fs.copyfile(results_file, self._fs.join(self._results_directory, "results.html"))
     1122
     1123    def _show_results_html_file(self, result_summary):
    11781124        """Shows the results.html page."""
     1125        if self._options.full_results_html:
     1126            test_files = result_summary.failures.keys()
     1127        else:
     1128            unexpected_failures = self._get_failures(result_summary, include_crashes=True)
     1129            test_files = unexpected_failures.keys()
     1130
     1131        if not len(test_files):
     1132            return
     1133
    11791134        results_filename = self._fs.join(self._results_directory, "results.html")
    11801135        self._port.show_results_html_file(results_filename)
Note: See TracChangeset for help on using the changeset viewer.