Changeset 84633 in webkit
- Timestamp:
- Apr 22, 2011 9:25:11 AM (13 years ago)
- Location:
- trunk/Tools
- Files:
-
- 2 added
- 5 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r84627 r84633 1 2011-04-21 Eric Seidel <eric@webkit.org> 2 3 Reviewed by Adam Barth. 4 5 LayoutTestResults should know how to handle NRWT json files 6 https://bugs.webkit.org/show_bug.cgi?id=59168 7 8 LayoutTestResults was originally created to be a parallel 9 to existing NRWT architecture for dealing with results files. 10 But then it turns out that NRWT has no such architecture. :( 11 So this patch writes the necessary code to handle reading 12 full_results.json and unexpected_results.json files from 13 NRWT layout-test-results directories. 14 15 LayoutTestResults has thus morphed from being ORWT-only to 16 supporting both ORWT and NRWT. It's possible at some future 17 point that other pieces of the NRWT architecture will learn 18 how to read JSON files in which case this can go away. 19 20 This is all done for making it possible for the commit-queue 21 to run the tests using NWRT and be able to understand the results 22 (for flaky test reporting, etc.) 23 24 * Scripts/webkitpy/common/net/layouttestresults.py: 25 * Scripts/webkitpy/common/net/layouttestresults_unittest.py: 26 * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py: 27 * Scripts/webkitpy/layout_tests/layout_package/test_expectations.py: 28 * Scripts/webkitpy/common/net/resultsjsonparser.py: Added. 29 * Scripts/webkitpy/common/net/resultsjsonparser_unittest.py: Added. 30 1 31 2011-04-22 Yi Shen <yi.4.shen@nokia.com> 2 32 -
trunk/Tools/Scripts/webkitpy/common/net/layouttestresults.py
r83876 r84633 30 30 # This class is one big hack and only needs to exist until we transition to new-run-webkit-tests. 31 31 32 from webkitpy.common.net.resultsjsonparser import ResultsJSONParser 32 33 from webkitpy.common.system.deprecated_logging import log 33 34 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, SoupStrainer … … 42 43 43 44 44 # FIXME: This should be unified with all the layout test results code in the layout_tests package 45 # This doesn't belong in common.net, but we don't have a better place for it yet. 46 class LayoutTestResults(object): 45 class ORWTResultsHTMLParser(object): 47 46 """This class knows how to parse old-run-webkit-tests results.html files.""" 48 47 … … 120 119 121 120 @classmethod 122 def _parse_results_html(cls, page):121 def parse_results_html(cls, page): 123 122 tables = BeautifulSoup(page).findAll("table") 124 123 return sum([cls._parse_results_table(table) for table in tables], []) 125 124 125 126 # FIXME: This should be unified with ResultsSummary or other NRWT layout tests code 127 # in the layout_tests package. 128 # This doesn't belong in common.net, but we don't have a better place for it yet. 129 class LayoutTestResults(object): 126 130 @classmethod 127 131 def results_from_string(cls, string): 128 132 if not string: 129 133 return None 130 test_results = cls._parse_results_html(string) 134 # For now we try to parse first as json, then as results.html 135 # eventually we will remove the html fallback support. 136 test_results = ResultsJSONParser.parse_results_json(string) 137 if not test_results: 138 test_results = ORWTResultsHTMLParser.parse_results_html(string) 131 139 if not test_results: 132 140 return None -
trunk/Tools/Scripts/webkitpy/common/net/layouttestresults_unittest.py
r83876 r84633 29 29 import unittest 30 30 31 from webkitpy.common.net.layouttestresults import LayoutTestResults 31 from webkitpy.common.net.layouttestresults import LayoutTestResults, ORWTResultsHTMLParser 32 32 from webkitpy.common.system.outputcapture import OutputCapture 33 33 from webkitpy.layout_tests.layout_package import test_results … … 36 36 37 37 38 class LayoutTestResultsTest(unittest.TestCase):38 class ORWTResultsHTMLParserTest(unittest.TestCase): 39 39 _example_results_html = """ 40 40 <html> … … 62 62 """ 63 63 64 def test_parse_layout_test_results(self): 65 failures = [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()] 66 testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html' 67 expected_results = [test_results.TestResult(testname, failures)] 68 69 results = ORWTResultsHTMLParser.parse_results_html(self._example_results_html) 70 self.assertEqual(expected_results, results) 71 72 73 def test_failures_from_fail_row(self): 74 row = BeautifulSoup("<tr><td><a>test.hml</a></td><td><a>expected image</a></td><td><a>25%</a></td></tr>") 75 test_name = unicode(row.find("a").string) 76 # Even if the caller has already found the test name, findAll inside _failures_from_fail_row will see it again. 77 failures = OutputCapture().assert_outputs(self, ORWTResultsHTMLParser._failures_from_fail_row, [row]) 78 self.assertEqual(len(failures), 1) 79 self.assertEqual(type(sorted(failures)[0]), test_failures.FailureImageHashMismatch) 80 81 row = BeautifulSoup("<tr><td><a>test.hml</a><a>foo</a></td></tr>") 82 expected_stderr = "Unhandled link text in results.html parsing: foo. Please file a bug against webkitpy.\n" 83 OutputCapture().assert_outputs(self, ORWTResultsHTMLParser._failures_from_fail_row, [row], expected_stderr=expected_stderr) 84 85 86 class LayoutTestResultsTest(unittest.TestCase): 87 64 88 def test_set_failure_limit_count(self): 65 89 results = LayoutTestResults([]) … … 68 92 self.assertEquals(results.failure_limit_count(), 10) 69 93 70 def test_parse_layout_test_results(self):71 failures = [test_failures.FailureMissingResult(), test_failures.FailureMissingImageHash(), test_failures.FailureMissingImage()]72 testname = 'fast/repaint/no-caret-repaint-in-non-content-editable-element.html'73 expected_results = [test_results.TestResult(testname, failures)]74 75 results = LayoutTestResults._parse_results_html(self._example_results_html)76 self.assertEqual(expected_results, results)77 78 94 def test_results_from_string(self): 79 95 self.assertEqual(LayoutTestResults.results_from_string(None), None) 80 96 self.assertEqual(LayoutTestResults.results_from_string(""), None) 81 results = LayoutTestResults.results_from_string( self._example_results_html)97 results = LayoutTestResults.results_from_string(ORWTResultsHTMLParserTest._example_results_html) 82 98 self.assertEqual(len(results.failing_tests()), 0) 83 84 def test_failures_from_fail_row(self):85 row = BeautifulSoup("<tr><td><a>test.hml</a></td><td><a>expected image</a></td><td><a>25%</a></td></tr>")86 test_name = unicode(row.find("a").string)87 # Even if the caller has already found the test name, findAll inside _failures_from_fail_row will see it again.88 failures = OutputCapture().assert_outputs(self, LayoutTestResults._failures_from_fail_row, [row])89 self.assertEqual(len(failures), 1)90 self.assertEqual(type(sorted(failures)[0]), test_failures.FailureImageHashMismatch)91 92 row = BeautifulSoup("<tr><td><a>test.hml</a><a>foo</a></td></tr>")93 expected_stderr = "Unhandled link text in results.html parsing: foo. Please file a bug against webkitpy.\n"94 OutputCapture().assert_outputs(self, LayoutTestResults._failures_from_fail_row, [row], expected_stderr=expected_stderr) -
trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
r80197 r84633 47 47 48 48 49 def has_json_wrapper(string): 50 return string.startswith(_JSON_PREFIX) and string.endswith(_JSON_SUFFIX) 51 52 49 53 def strip_json_wrapper(json_content): 50 54 return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)] -
trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
r83475 r84633 42 42 43 43 # Test expectation and modifier constants. 44 # FIXME: range() starts with 0 which makes if expectation checks harder 45 # as PASS is 0. 44 46 (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX, 45 47 SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(16) … … 148 150 raise ValueError(expectation) 149 151 152 @classmethod 153 def expectation_from_string(cls, string): 154 assert(' ' not in string) # This only handles one expectation at a time. 155 return TestExpectationsFile.EXPECTATIONS.get(string.lower()) 156 150 157 def get_tests_with_result_type(self, result_type): 151 158 return self._expected_failures.get_tests_with_result_type(result_type) … … 532 539 result = set() 533 540 for part in expectations: 534 if not part in self.EXPECTATIONS:535 self._add_error(lineno, 'Unsupported expectation: %s' % part,536 541 expectation = TestExpectations.expectation_from_string(part) 542 if expectation is None: # Careful, PASS is currently 0. 543 self._add_error(lineno, 'Unsupported expectation: %s' % part, test_list_path) 537 544 continue 538 expectation = self.EXPECTATIONS[part]539 545 result.add(expectation) 540 546 return result
Note: See TracChangeset
for help on using the changeset viewer.