Changeset 120740 in webkit
- Timestamp:
- Jun 19, 2012 11:27:29 AM (12 years ago)
- Location:
- trunk/Tools
- Files:
-
- 5 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r120738 r120740 1 2012-06-19 Dirk Pranke <dpranke@chromium.org> 2 3 REGRESSION (NRWT): Results for new non-text-only tests are always put in the most-specific platform directory 4 https://bugs.webkit.org/show_bug.cgi?id=78127 5 6 Reviewed by Ryosuke Niwa. 7 8 Add the --add-platform-exceptions flag from ORWT, and ensure 9 that --new-baseline is equivalent to --reset-results 10 --add-platform-exceptions. Also fix the default logic for where to 11 put PNGs and render tree dumps if --new-test-results is true: 12 if --add-platform-exceptions is False, baselines should go in 13 the least-specific platform directory (e.g., platform/mac) 14 rather than the most-specific (platform/mac-snowleopard). 15 16 * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py: 17 (SingleTestRunner): 18 (SingleTestRunner.run): 19 (SingleTestRunner._run_rebaseline): 20 (SingleTestRunner._add_missing_baselines): 21 (SingleTestRunner._location_for_new_baseline): 22 (SingleTestRunner._overwrite_baselines): 23 (SingleTestRunner._save_baseline_data): 24 * Scripts/webkitpy/layout_tests/port/base.py: 25 (Port.baseline_path): 26 (Port): 27 (Port.baseline_platform_dir): 28 (Port.baseline_version_dir): 29 * Scripts/webkitpy/layout_tests/run_webkit_tests.py: 30 (_set_up_derived_options): 31 (parse_args): 32 * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py: 33 (RebaselineTest.assertBaselines): 34 (RebaselineTest.test_reset_results): 35 (RebaselineTest.test_missing_results): 36 (RebaselineTest.test_new_baseline): 37 1 38 2012-06-19 Dirk Pranke <dpranke@chromium.org> 2 39 -
trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
r116600 r120740 47 47 48 48 49 class SingleTestRunner: 49 class SingleTestRunner(object): 50 (ALONGSIDE_TEST, PLATFORM_DIR, VERSION_DIR, UPDATE) = ('alongside', 'platform', 'version', 'update') 50 51 51 52 def __init__(self, options, port, driver, test_input, worker_name): … … 95 96 def run(self): 96 97 if self._reference_files: 97 if self._port.get_option('no_ref_tests') or self._options. new_baseline or self._options.reset_results:98 if self._port.get_option('no_ref_tests') or self._options.reset_results: 98 99 result = TestResult(self._test_name) 99 100 result.type = test_expectations.SKIP 100 101 return result 101 102 return self._run_reftest() 102 if self._options. new_baseline or self._options.reset_results:103 if self._options.reset_results: 103 104 return self._run_rebaseline() 104 105 return self._run_compare_test() … … 122 123 failures = self._handle_error(driver_output) 123 124 test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, None, failures) 124 # FIXME: It the test crashed or timed out, it might be bett ter to avoid125 # FIXME: It the test crashed or timed out, it might be better to avoid 125 126 # to write new baselines. 126 127 self._overwrite_baselines(driver_output) … … 132 133 missingImage = test_result.has_failure_matching_types(test_failures.FailureMissingImage, test_failures.FailureMissingImageHash) 133 134 if test_result.has_failure_matching_types(test_failures.FailureMissingResult): 134 self._save_baseline_data(driver_output.text, ".txt", SingleTestRunner._render_tree_dump_pattern.match(driver_output.text))135 self._save_baseline_data(driver_output.text, '.txt', self._location_for_new_baseline(driver_output.text, '.txt')) 135 136 if test_result.has_failure_matching_types(test_failures.FailureMissingAudio): 136 self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=False)137 self._save_baseline_data(driver_output.audio, '.wav', self._location_for_new_baseline(driver_output.audio, '.wav')) 137 138 if missingImage: 138 self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=True) 139 self._save_baseline_data(driver_output.image, '.png', self._location_for_new_baseline(driver_output.image, '.png')) 140 141 def _location_for_new_baseline(self, data, extension): 142 if self._options.add_platform_exceptions: 143 return self.VERSION_DIR 144 if extension == '.png': 145 return self.PLATFORM_DIR 146 if extension == '.wav': 147 return self.ALONGSIDE_TEST 148 if extension == '.txt' and self._render_tree_dump_pattern.match(data): 149 return self.PLATFORM_DIR 150 return self.ALONGSIDE_TEST 139 151 140 152 def _overwrite_baselines(self, driver_output): 141 # Although all DumpRenderTree output should be utf-8, 142 # we do not ever decode it inside run-webkit-tests. For some tests 143 # DumpRenderTree may not output utf-8 text (e.g. webarchives). 144 self._save_baseline_data(driver_output.text, ".txt", generate_new_baseline=self._options.new_baseline) 145 self._save_baseline_data(driver_output.audio, ".wav", generate_new_baseline=self._options.new_baseline) 153 location = self.VERSION_DIR if self._options.add_platform_exceptions else self.UPDATE 154 self._save_baseline_data(driver_output.text, '.txt', location) 155 self._save_baseline_data(driver_output.audio, '.wav', location) 146 156 if self._options.pixel_tests: 147 self._save_baseline_data(driver_output.image, ".png", generate_new_baseline=self._options.new_baseline) 148 149 def _save_baseline_data(self, data, modifier, generate_new_baseline=True): 150 """Saves a new baseline file into the port's baseline directory. 151 152 The file will be named simply "<test>-expected<modifier>", suitable for 153 use as the expected results in a later run. 154 155 Args: 156 data: result to be saved as the new baseline 157 modifier: type of the result file, e.g. ".txt" or ".png" 158 generate_new_baseline: whether to enerate a new, platform-specific 159 baseline, or update the existing one 160 """ 157 self._save_baseline_data(driver_output.image, '.png', location) 158 159 def _save_baseline_data(self, data, extension, location): 161 160 if data is None: 162 161 return 163 162 port = self._port 164 163 fs = self._filesystem 165 if generate_new_baseline: 166 relative_dir = fs.dirname(self._test_name) 167 baseline_path = port.baseline_path() 168 output_dir = fs.join(baseline_path, relative_dir) 169 output_file = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + modifier) 170 fs.maybe_make_directory(output_dir) 171 output_path = fs.join(output_dir, output_file) 164 if location == self.ALONGSIDE_TEST: 165 output_dir = fs.dirname(port.abspath_for_test(self._test_name)) 166 elif location == self.VERSION_DIR: 167 output_dir = fs.join(port.baseline_version_dir(), fs.dirname(self._test_name)) 168 elif location == self.PLATFORM_DIR: 169 output_dir = fs.join(port.baseline_platform_dir(), fs.dirname(self._test_name)) 170 elif location == self.UPDATE: 171 output_dir = fs.dirname(port.expected_filename(self._test_name, extension)) 172 172 else: 173 output_path = port.expected_filename(self._test_name, modifier) 174 175 result_name = fs.relpath(output_path, port.layout_tests_dir()) 176 _log.info('Writing new expected result "%s"' % result_name) 173 raise AssertionError('unrecognized baseline location: %s' % location) 174 175 fs.maybe_make_directory(output_dir) 176 output_basename = fs.basename(fs.splitext(self._test_name)[0] + "-expected" + extension) 177 output_path = fs.join(output_dir, output_basename) 178 _log.info('Writing new expected result "%s"' % port.relative_test_filename(output_path)) 177 179 port.update_baseline(output_path, data) 178 180 -
trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py
r120738 r120740 166 166 def baseline_path(self): 167 167 """Return the absolute path to the directory to store new baselines in for this port.""" 168 # FIXME: remove once all callers are calling either baseline_version_dir() or baseline_platform_dir() 169 return self.baseline_version_dir() 170 171 def baseline_platform_dir(self): 172 """Return the absolute path to the default (version-independent) platform-specific results.""" 173 return self._filesystem.join(self.layout_tests_dir(), 'platform', self.port_name) 174 175 def baseline_version_dir(self): 176 """Return the absolute path to the platform-and-version-specific results.""" 168 177 baseline_search_paths = self.get_option('additional_platform_directory', []) + self.baseline_search_path() 169 178 return baseline_search_paths[0] -
trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
r120646 r120740 168 168 warnings.append("--ignore-metrics has no effect with --new-baselines or with --reset-results") 169 169 170 if options.new_baseline: 171 options.reset_results = True 172 options.add_platform_exceptions = True 173 170 174 return warnings 171 175 … … 278 282 optparse.make_option("--build-directory", 279 283 help="Path to the directory under which build files are kept (should not include configuration)"), 284 optparse.make_option("--add-platform-exceptions", action="store_true", default=False, 285 help="Save generated results into the *most-specific-platform* directory rather than the *generic-platform* directory"), 280 286 optparse.make_option("--new-baseline", action="store_true", 281 287 default=False, help="Save generated results as new baselines " 282 "into the * platform* directory, overwriting whatever's "283 "already there. "),288 "into the *most-specific-platform* directory, overwriting whatever's " 289 "already there. Equivalent to --reset-results --add-platform-exceptions"), 284 290 optparse.make_option("--reset-results", action="store_true", 285 291 default=False, help="Reset expectations to the " -
trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py
r120349 r120740 931 931 for ext in extensions: 932 932 baseline = file + "-expected" + ext 933 baseline_msg = 'Writing new expected result "%s"\n' % baseline [1:]933 baseline_msg = 'Writing new expected result "%s"\n' % baseline 934 934 self.assertTrue(any(f.find(baseline) != -1 for f in file_list)) 935 935 self.assertContainsLine(err, baseline_msg) … … 952 952 self.assertEmpty(out) 953 953 self.assertEqual(len(file_list), 4) 954 self.assertBaselines(file_list, " /passes/image", [".txt", ".png"], err)955 self.assertBaselines(file_list, " /failures/expected/missing_image", [".txt", ".png"], err)954 self.assertBaselines(file_list, "passes/image", [".txt", ".png"], err) 955 self.assertBaselines(file_list, "failures/expected/missing_image", [".txt", ".png"], err) 956 956 957 957 def test_missing_results(self): … … 970 970 self.assertNotEmpty(out) 971 971 self.assertEqual(len(file_list), 6) 972 self.assertBaselines(file_list, " /failures/unexpected/missing_text", [".txt"], err)973 self.assertBaselines(file_list, " /platform/test-mac-leopard/failures/unexpected/missing_image", [".png"], err)974 self.assertBaselines(file_list, " /platform/test-mac-leopard/failures/unexpected/missing_render_tree_dump", [".txt"], err)972 self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err) 973 self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err) 974 self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_render_tree_dump", [".txt"], err) 975 975 976 976 def test_new_baseline(self): 977 # Test that we update the platform expectations . If the expectation978 # is mssing, then create a new expectation in the platform dir.977 # Test that we update the platform expectations in the version-specific directories 978 # for both existing and new baselines. 979 979 host = MockHost() 980 980 res, out, err, _ = logging_run(['--pixel-tests', … … 989 989 self.assertEqual(len(file_list), 4) 990 990 self.assertBaselines(file_list, 991 " /platform/test-mac-leopard/passes/image", [".txt", ".png"], err)991 "platform/test-mac-leopard/passes/image", [".txt", ".png"], err) 992 992 self.assertBaselines(file_list, 993 " /platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)993 "platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err) 994 994 995 995
Note: See TracChangeset
for help on using the changeset viewer.