Changeset 239989 in webkit
- Timestamp:
- Jan 15, 2019 9:46:31 AM (5 years ago)
- Location:
- trunk/Tools
- Files:
-
- 19 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r239983 r239989 1 2019-01-15 Jonathan Bedard <jbedard@apple.com> 2 3 webkitpy: Implement device type specific expected results (Part 1) 4 https://bugs.webkit.org/show_bug.cgi?id=192162 5 6 Reviewed by Lucas Forschler. 7 8 Device type specific expected results covers two related ideas. The first (covered by this change) 9 is where to search for -expected.* files. The second is which tests to run based on the 10 TestExpectations.txt file. 11 12 After this change, the baseline search path will be determined by the type of device running a 13 test. Device type will be passed into any function which picks a file based on the baseline search 14 path. 15 16 * Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py: 17 (Worker._update_test_input): Pass device_type to self._port.reference_files(...). 18 * Scripts/webkitpy/layout_tests/controllers/manager.py: 19 (Manager.run): Print baseline search path for each round of devices. 20 * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py: 21 (SingleTestRunner.__init__): Pass device_type into expected_*(...). 22 (SingleTestRunner._expected_driver_output): Ditto. 23 (SingleTestRunner._driver_input): Ditto. 24 * Scripts/webkitpy/layout_tests/views/printing.py: 25 (Printer.print_config): Move printing of baseline search path to a separate function. 26 (Printer.print_baseline_search_path): Moved from Printer.print_config. 27 * Scripts/webkitpy/layout_tests/views/printing_unittest.py: 28 (Testprinter.test_print_config): 29 (Testprinter.test_print_baseline_search_path): 30 * Scripts/webkitpy/port/base.py: 31 (Port.baseline_search_path): Pass device_type through. 32 (Port._expected_baselines_for_suffixes): Ditto. 33 (Port.expected_baselines): Ditto. 34 (Port.expected_filename): Ditto. 35 (Port.expected_checksum): Ditto. 36 (Port.expected_image): Ditto. 37 (Port.expected_audio): Ditto. 38 (Port.expected_text): Ditto. 39 (Port.reference_files): Ditto. 40 (Port.tests): Ditto. 41 (Port._expanded_paths): Ditto. 42 (Port.skipped_layout_tests): Ditto. 43 (Port.expectations_dict): Ditto. 44 (Port._port_specific_expectations_files): Ditto. 45 (Port.expectations_files): Ditto. 46 (Port.test_search_path): Ditto. 47 (Port._tests_for_other_platforms): Ditto. 48 * Scripts/webkitpy/port/base_unittest.py: 49 (PortTest.test_additional_platform_directory): Accept, but do not use, device_type. 50 (PortTest.test_nonexistant_expectations): Ditto. 51 (test_ref_tests_platform_directory): Ditto. 52 * Scripts/webkitpy/port/darwin.py: 53 (DarwinPort._port_specific_expectations_files): Accept device_type. 54 * Scripts/webkitpy/port/driver.py: 55 (DriverProxy): 56 (DriverProxy.host): Expose _target_host for device_type information. 57 * Scripts/webkitpy/port/gtk.py: 58 (GtkPort._port_specific_expectations_files): Accept, but do not use, device_type. 59 * Scripts/webkitpy/port/ios.py: 60 (IOSPort. default_baseline_search_path): Remove memoized, since it does not accept **kwargs. 61 * Scripts/webkitpy/port/mac.py: 62 (MacPort. default_baseline_search_path): Ditto. 63 * Scripts/webkitpy/port/port_testcase.py: 64 (TestWebKitPort._tests_for_other_platforms): Accept, but do not use, device_type. 65 * Scripts/webkitpy/port/test.py: 66 * Scripts/webkitpy/port/watch.py: 67 (WatchPort. default_baseline_search_path): Remove memoized, since it does not accept **kwargs. 68 * Scripts/webkitpy/port/win.py: 69 (WinPort.test_search_path): Accept, but do not use, device_type. 70 * Scripts/webkitpy/port/wpe.py: 71 (WPEPort._port_specific_expectations_files): Ditto. 72 * Scripts/webkitpy/tool/servers/rebaselineserver.py: 73 (get_test_baselines.AllPlatformsPort.baseline_search_path): Ditto. 74 1 75 2019-01-15 Zalan Bujtas <zalan@apple.com> 2 76 -
trunk/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py
r235771 r239989 295 295 if test_input.reference_files is None: 296 296 # Lazy initialization. 297 test_input.reference_files = self._port.reference_files(test_input.test_name )297 test_input.reference_files = self._port.reference_files(test_input.test_name, device_type=self._port.target_host(self._worker_number).device_type) 298 298 if test_input.reference_files: 299 299 test_input.should_run_pixel_test = True -
trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
r239739 r239989 275 275 index += 1 276 276 277 self._printer.print_baseline_search_path(device_type=device_type) 278 277 279 _log.info('Running {}{}'.format(pluralize(len(tests), 'test'), ' for {}'.format(str(device_type)) if device_type else '')) 278 280 _log.info('') -
trunk/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
r217130 r239989 70 70 # in either layout tests or reftests, but not in both. 71 71 for suffix in ('.txt', '.png', '.wav'): 72 expected_filename = self._port.expected_filename(self._test_name, suffix )72 expected_filename = self._port.expected_filename(self._test_name, suffix, device_type=self._driver.host.device_type) 73 73 if self._filesystem.exists(expected_filename): 74 74 _log.error('%s is a reftest, but has an unused expectation file. Please remove %s.', self._test_name, expected_filename) 75 75 76 76 def _expected_driver_output(self): 77 return DriverOutput(self._port.expected_text(self._test_name ),78 self._port.expected_image(self._test_name ),79 self._port.expected_checksum(self._test_name ),80 self._port.expected_audio(self._test_name ))77 return DriverOutput(self._port.expected_text(self._test_name, device_type=self._driver.host.device_type), 78 self._port.expected_image(self._test_name, device_type=self._driver.host.device_type), 79 self._port.expected_checksum(self._test_name, device_type=self._driver.host.device_type), 80 self._port.expected_audio(self._test_name, device_type=self._driver.host.device_type)) 81 81 82 82 def _should_fetch_expected_checksum(self): … … 90 90 image_hash = None 91 91 if self._should_fetch_expected_checksum(): 92 image_hash = self._port.expected_checksum(self._test_name )92 image_hash = self._port.expected_checksum(self._test_name, device_type=self._driver.host.device_type) 93 93 return DriverInput(self._test_name, self._timeout, image_hash, self._should_run_pixel_test, self._should_dump_jsconsolelog_in_stderr) 94 94 -
trunk/Tools/Scripts/webkitpy/layout_tests/views/printing.py
r239874 r239989 82 82 self._print_default("Placing new baselines in %s" % self._port.baseline_path()) 83 83 84 fs = self._port.host.filesystem85 full_baseline_search_path = self._port.baseline_search_path()86 normalize_baseline = lambda baseline_search_path: [fs.relpath(x, self._port.layout_tests_dir()).replace("../", "") for x in baseline_search_path]87 88 self._print_default('Verbose baseline search path: {} -> generic'.format(' -> '.join(normalize_baseline(full_baseline_search_path))))89 90 self._print_default('')91 self._print_default('Baseline search path: {} -> generic'.format(' -> '.join(normalize_baseline([path for path in full_baseline_search_path if fs.exists(path)]))))92 self._print_default('')93 94 84 self._print_default("Using %s build" % self._options.configuration) 95 85 if self._options.pixel_tests: … … 102 92 103 93 self._print_default('Command line: ' + ' '.join(self._port.driver_cmd_line_for_logging())) 94 self._print_default('') 95 96 def print_baseline_search_path(self, device_type=None): 97 fs = self._port.host.filesystem 98 full_baseline_search_path = self._port.baseline_search_path(device_type=device_type) 99 normalize_baseline = lambda baseline_search_path: [ 100 fs.relpath(x, self._port.layout_tests_dir()).replace("../", "") for x in baseline_search_path] 101 102 self._print_default('Verbose baseline search path: {} -> generic'.format( 103 ' -> '.join(normalize_baseline(full_baseline_search_path)))) 104 105 self._print_default('') 106 self._print_default('Baseline search path: {} -> generic'.format( 107 ' -> '.join(normalize_baseline([path for path in full_baseline_search_path if fs.exists(path)])))) 104 108 self._print_default('') 105 109 -
trunk/Tools/Scripts/webkitpy/layout_tests/views/printing_unittest.py
r239874 r239989 112 112 self.assertIn('Test configuration: <leopard, x86, release>', err.getvalue()) 113 113 self.assertIn('Placing test results in /tmp', err.getvalue()) 114 self.assertIn('Verbose baseline search path: platform/test-mac-leopard -> platform/test-mac-snowleopard -> generic', err.getvalue())115 self.assertIn('Baseline search path: platform/test-mac-leopard -> generic', err.getvalue())116 114 self.assertIn('Using Release build', err.getvalue()) 117 115 self.assertIn('Pixel tests enabled', err.getvalue()) … … 119 117 self.assertIn('Regular timeout: ', err.getvalue()) 120 118 119 def test_print_baseline_search_path(self): 120 printer, err = self.get_printer() 121 printer.print_baseline_search_path() 122 123 self.assertIn('Verbose baseline search path: platform/test-mac-leopard -> platform/test-mac-snowleopard -> generic', err.getvalue()) 124 self.assertIn('Baseline search path: platform/test-mac-leopard -> generic', err.getvalue()) 125 121 126 self.reset(err) 122 127 printer._options.quiet = True 123 printer.print_ config('/tmp')128 printer.print_baseline_search_path() 124 129 self.assertNotIn('Baseline search path: platform/test-mac-leopard -> platform/test-mac-snowleopard -> generic', err.getvalue()) 125 130 -
trunk/Tools/Scripts/webkitpy/port/base.py
r239875 r239989 209 209 return baseline_search_paths[0] 210 210 211 def baseline_search_path(self ):212 return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path( )211 def baseline_search_path(self, device_type=None): 212 return self.get_option('additional_platform_directory', []) + self._compare_baseline() + self.default_baseline_search_path(device_type=device_type) 213 213 214 214 def default_baseline_search_path(self, device_type=None): … … 409 409 return ('.wav', '.webarchive', '.txt', '.png') 410 410 411 def _expected_baselines_for_suffixes(self, test_name, suffixes, all_baselines=False ):412 baseline_search_path = self.baseline_search_path( ) + [self.layout_tests_dir()]411 def _expected_baselines_for_suffixes(self, test_name, suffixes, all_baselines=False, device_type=None): 412 baseline_search_path = self.baseline_search_path(device_type=device_type) + [self.layout_tests_dir()] 413 413 414 414 baselines = [] … … 429 429 return baselines 430 430 431 def expected_baselines(self, test_name, suffix, all_baselines=False ):431 def expected_baselines(self, test_name, suffix, all_baselines=False, device_type=None): 432 432 """Given a test name, finds where the baseline results are located. 433 433 … … 456 456 platform specific. 457 457 """ 458 return self._expected_baselines_for_suffixes(test_name, [suffix], all_baselines=all_baselines )459 460 def expected_filename(self, test_name, suffix, return_default=True ):458 return self._expected_baselines_for_suffixes(test_name, [suffix], all_baselines=all_baselines, device_type=device_type) 459 460 def expected_filename(self, test_name, suffix, return_default=True, device_type=None): 461 461 """Given a test name, returns an absolute path to its expected results. 462 462 … … 478 478 the other baseline and filename manipulation routines. 479 479 """ 480 platform_dir, baseline_filename = self.expected_baselines(test_name, suffix )[0]480 platform_dir, baseline_filename = self.expected_baselines(test_name, suffix, device_type=device_type)[0] 481 481 if platform_dir or return_default: 482 482 return self._filesystem.join(platform_dir or self.layout_tests_dir(), baseline_filename) 483 483 return None 484 484 485 def expected_checksum(self, test_name ):485 def expected_checksum(self, test_name, device_type=None): 486 486 """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test.""" 487 png_path = self.expected_filename(test_name, '.png' )487 png_path = self.expected_filename(test_name, '.png', device_type=device_type) 488 488 489 489 if self._filesystem.exists(png_path): … … 493 493 return None 494 494 495 def expected_image(self, test_name ):495 def expected_image(self, test_name, device_type=None): 496 496 """Returns the image we expect the test to produce.""" 497 baseline_path = self.expected_filename(test_name, '.png' )497 baseline_path = self.expected_filename(test_name, '.png', device_type=device_type) 498 498 if not self._filesystem.exists(baseline_path): 499 499 return None 500 500 return self._filesystem.read_binary_file(baseline_path) 501 501 502 def expected_audio(self, test_name ):503 baseline_path = self.expected_filename(test_name, '.wav' )502 def expected_audio(self, test_name, device_type=None): 503 baseline_path = self.expected_filename(test_name, '.wav', device_type=device_type) 504 504 if not self._filesystem.exists(baseline_path): 505 505 return None 506 506 return self._filesystem.read_binary_file(baseline_path) 507 507 508 def expected_text(self, test_name ):508 def expected_text(self, test_name, device_type=None): 509 509 """Returns the text output we expect the test to produce, or None 510 510 if we don't expect there to be any text output. … … 513 513 # output from DRT (instead treating it as a binary string), we read the 514 514 # baselines as a binary string, too. 515 baseline_path = self.expected_filename(test_name, '.txt' )515 baseline_path = self.expected_filename(test_name, '.txt', device_type=device_type) 516 516 if not self._filesystem.exists(baseline_path): 517 baseline_path = self.expected_filename(test_name, '.webarchive' )517 baseline_path = self.expected_filename(test_name, '.webarchive', device_type=device_type) 518 518 if not self._filesystem.exists(baseline_path): 519 519 return None … … 544 544 return parsed_list 545 545 546 def reference_files(self, test_name ):546 def reference_files(self, test_name, device_type=None): 547 547 """Return a list of expectation (== or !=) and filename pairs""" 548 548 … … 559 559 for part2 in self._supported_reference_extensions: 560 560 suffixes.append(part1 + part2) 561 for platform_dir, baseline_filename in self._expected_baselines_for_suffixes(test_name, suffixes ):561 for platform_dir, baseline_filename in self._expected_baselines_for_suffixes(test_name, suffixes, device_type=device_type): 562 562 if not platform_dir: 563 563 continue … … 580 580 return [self.host.filesystem.relpath(test, self.layout_tests_dir()) for test in self._filesystem.glob(re.sub('-expected.*', '.*', self._filesystem.join(self.layout_tests_dir(), path))) if self._filesystem.isfile(test)] 581 581 582 def tests(self, paths ):582 def tests(self, paths, device_type=None): 583 583 """Return the list of tests found. Both generic and platform-specific tests matching paths should be returned.""" 584 expanded_paths = self._expanded_paths(paths )584 expanded_paths = self._expanded_paths(paths, device_type=device_type) 585 585 return self._real_tests(expanded_paths) 586 586 587 def _expanded_paths(self, paths ):587 def _expanded_paths(self, paths, device_type=None): 588 588 expanded_paths = [] 589 589 fs = self._filesystem … … 593 593 if self.test_isdir(path) and not path.startswith('platform') and not fs.isabs(path): 594 594 for platform_dir in all_platform_dirs: 595 if fs.isdir(fs.join(platform_dir, path)) and platform_dir in self.baseline_search_path( ):595 if fs.isdir(fs.join(platform_dir, path)) and platform_dir in self.baseline_search_path(device_type=device_type): 596 596 expanded_paths.append(self.relative_test_filename(fs.join(platform_dir, path))) 597 597 … … 754 754 return self._webkit_finder.perf_tests_dir() 755 755 756 def skipped_layout_tests(self, test_list ):756 def skipped_layout_tests(self, test_list, device_type=None): 757 757 """Returns tests skipped outside of the TestExpectations files.""" 758 return set(self._tests_for_other_platforms( )).union(self._skipped_tests_for_unsupported_features(test_list))758 return set(self._tests_for_other_platforms(device_type=device_type)).union(self._skipped_tests_for_unsupported_features(test_list)) 759 759 760 760 @memoized … … 1160 1160 return False 1161 1161 1162 def expectations_dict(self ):1162 def expectations_dict(self, device_type=None): 1163 1163 """Returns an OrderedDict of name -> expectations strings. 1164 1164 The names are expected to be (but not required to be) paths in the filesystem. … … 1171 1171 expectations = OrderedDict() 1172 1172 1173 for path in self.expectations_files( ):1173 for path in self.expectations_files(device_type=device_type): 1174 1174 if self._filesystem.exists(path): 1175 1175 expectations[path] = self._filesystem.read_text_file(path) … … 1184 1184 return expectations 1185 1185 1186 def _port_specific_expectations_files(self ):1186 def _port_specific_expectations_files(self, **kwargs): 1187 1187 # Unlike baseline_search_path, we only want to search [WK2-PORT, PORT-VERSION, PORT] and any directories 1188 1188 # included via --additional-platform-directory, not the full casade. … … 1202 1202 return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpectations') for d in search_paths] 1203 1203 1204 def expectations_files(self ):1205 return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files( )1204 def expectations_files(self, device_type=None): 1205 return [self.path_to_generic_test_expectations_file()] + self._port_specific_expectations_files(device_type=device_type) 1206 1206 1207 1207 def repository_paths(self): … … 1535 1535 return [] 1536 1536 1537 def test_search_path(self ):1538 return self.baseline_search_path( )1539 1540 def _tests_for_other_platforms(self ):1537 def test_search_path(self, device_type=None): 1538 return self.baseline_search_path(device_type=device_type) 1539 1540 def _tests_for_other_platforms(self, device_type=None): 1541 1541 # By default we will skip any directory under LayoutTests/platform 1542 1542 # that isn't in our baseline search path (this mirrors what … … 1546 1546 dirs_to_skip = [] 1547 1547 for entry in entries: 1548 if self._filesystem.isdir(entry) and entry not in self.test_search_path( ):1548 if self._filesystem.isdir(entry) and entry not in self.test_search_path(device_type=device_type): 1549 1549 basename = self._filesystem.basename(entry) 1550 1550 dirs_to_skip.append('platform/%s' % basename) -
trunk/Tools/Scripts/webkitpy/port/base_unittest.py
r239048 r239989 162 162 def test_additional_platform_directory(self): 163 163 port = self.make_port(port_name='foo') 164 port.default_baseline_search_path = lambda : ['LayoutTests/platform/foo']164 port.default_baseline_search_path = lambda **kwargs: ['LayoutTests/platform/foo'] 165 165 test_file = 'fast/test.html' 166 166 … … 188 188 def test_nonexistant_expectations(self): 189 189 port = self.make_port(port_name='foo') 190 port.expectations_files = lambda : ['/mock-checkout/LayoutTests/platform/exists/TestExpectations', '/mock-checkout/LayoutTests/platform/nonexistant/TestExpectations']190 port.expectations_files = lambda **kwargs: ['/mock-checkout/LayoutTests/platform/exists/TestExpectations', '/mock-checkout/LayoutTests/platform/nonexistant/TestExpectations'] 191 191 port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/exists/TestExpectations', '') 192 192 self.assertEqual('\n'.join(port.expectations_dict().keys()), '/mock-checkout/LayoutTests/platform/exists/TestExpectations') … … 414 414 def test_ref_tests_platform_directory(self): 415 415 port = self.make_port(port_name='foo') 416 port.default_baseline_search_path = lambda : ['/mock-checkout/LayoutTests/platform/foo']416 port.default_baseline_search_path = lambda **kwargs: ['/mock-checkout/LayoutTests/platform/foo'] 417 417 port._filesystem.write_text_file('/mock-checkout/LayoutTests/fast/ref-expected.html', 'foo') 418 418 -
trunk/Tools/Scripts/webkitpy/port/darwin.py
r229085 r239989 54 54 return super(DarwinPort, self).default_timeout_ms() 55 55 56 def _port_specific_expectations_files(self ):57 return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path( )]))56 def _port_specific_expectations_files(self, device_type=None): 57 return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self.baseline_search_path(device_type=device_type)])) 58 58 59 59 def check_for_leaks(self, process_name, process_pid): -
trunk/Tools/Scripts/webkitpy/port/driver.py
r235467 r239989 720 720 return self._driver_instance_constructor(self._port, self._worker_number, pixel_tests, self._no_timeout) 721 721 722 @property 723 def host(self): 724 return self._driver._target_host 725 722 726 # FIXME: this should be a @classmethod (or implemented on Port instead). 723 727 def is_http_test(self, test_name): -
trunk/Tools/Scripts/webkitpy/port/gtk.py
r239875 r239989 214 214 return map(self._webkit_baseline_path, self._search_paths()) 215 215 216 def _port_specific_expectations_files(self ):216 def _port_specific_expectations_files(self, **kwargs): 217 217 return [self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in reversed(self._search_paths())] 218 218 -
trunk/Tools/Scripts/webkitpy/port/ios.py
r239875 r239989 53 53 return VersionNameMap.map(self.host.platform).to_name(self._os_version, platform=IOSPort.port_name) 54 54 55 @memoized56 55 def default_baseline_search_path(self, device_type=None): 57 56 wk_string = 'wk1' -
trunk/Tools/Scripts/webkitpy/port/mac.py
r239875 r239989 70 70 return ['ARCHS=i386'] if self.architecture() == 'x86' else [] 71 71 72 @memoized73 72 def default_baseline_search_path(self, **kwargs): 74 73 versions_to_fallback = [] -
trunk/Tools/Scripts/webkitpy/port/port_testcase.py
r238944 r239989 71 71 return self.symbols_string 72 72 73 def _tests_for_other_platforms(self ):73 def _tests_for_other_platforms(self, **kwargs): 74 74 return ["media", ] 75 75 -
trunk/Tools/Scripts/webkitpy/port/test.py
r238903 r239989 408 408 return 'MOCK _path_to_driver' 409 409 410 def baseline_search_path(self ):410 def baseline_search_path(self, **kwargs): 411 411 search_paths = { 412 412 'test-mac-snowleopard': ['test-mac-snowleopard'], -
trunk/Tools/Scripts/webkitpy/port/watch.py
r239875 r239989 59 59 return 4 60 60 61 @memoized62 61 def default_baseline_search_path(self, **kwargs): 63 62 versions_to_fallback = [] -
trunk/Tools/Scripts/webkitpy/port/win.py
r239875 r239989 218 218 return {binary.split('.')[0]: self._build_path(binary) for binary in self.API_TEST_BINARY_NAMES} 219 219 220 def test_search_path(self ):220 def test_search_path(self, **kwargs): 221 221 test_fallback_names = [path for path in self.baseline_search_path() if not path.startswith(self._webkit_baseline_path('mac'))] 222 222 return map(self._webkit_baseline_path, test_fallback_names) -
trunk/Tools/Scripts/webkitpy/port/wpe.py
r239875 r239989 106 106 return map(self._webkit_baseline_path, self._search_paths()) 107 107 108 def _port_specific_expectations_files(self ):108 def _port_specific_expectations_files(self, **kwargs): 109 109 return map(lambda x: self._filesystem.join(self._webkit_baseline_path(x), 'TestExpectations'), reversed(self._search_paths())) 110 110 -
trunk/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py
r225733 r239989 169 169 self._platforms_by_directory = dict([(self._webkit_baseline_path(p), p) for p in test_config.platforms]) 170 170 171 def baseline_search_path(self ):171 def baseline_search_path(self, **kwargs): 172 172 return self._platforms_by_directory.keys() 173 173
Note: See TracChangeset
for help on using the changeset viewer.