Changeset 238903 in webkit
- Timestamp:
- Dec 5, 2018 11:30:47 AM (5 years ago)
- Location:
- trunk/Tools
- Files:
-
- 13 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r238901 r238903 1 2018-12-05 Jonathan Bedard <jbedard@apple.com> 2 3 webkitpy: Sort tests by associated device type 4 https://bugs.webkit.org/show_bug.cgi?id=192161 5 <rdar://problem/46345392> 6 7 Reviewed by Lucas Forschler. 8 9 Sort tests by device type and make an effort to run each specified device type. 10 Do not run tests if their specified device is not available. 11 12 * Scripts/webkitpy/common/system/platforminfo.py: 13 (PlatformInfo.is_watchos): Add watchos bits to platform info. 14 (PlatformInfo._determine_os_name): Ditto. 15 * Scripts/webkitpy/layout_tests/controllers/manager.py: 16 (Manager.run): Assign each test a device type. Then, generate a list of 17 device types to sequentially iterate through. Note that a test will run 18 on the first device which matches. 19 (Manager._end_test_run): Handle the case where no devices are available and no 20 tests are run. 21 * Scripts/webkitpy/layout_tests/run_webkit_tests.py: 22 (_set_up_derived_options): Manage child processes in manager. 23 * Scripts/webkitpy/port/base.py: 24 (Port): 25 (Port.default_child_processes): Accept additional arguments. 26 (Port.max_child_processes): Add upper limit for number of child processes. 27 * Scripts/webkitpy/port/device_port.py: 28 (DevicePort): 29 (DevicePort._device_type_with_version): Adds version to the DeviceType. 30 (DevicePort.default_child_processes): Allows default_child_processes to be 31 attached to a device type. 32 (DevicePort.max_child_processes): Add upper limit to the maximum number of child processes. 33 (DevicePort.setup_test_run): Use _device_type_with_version. 34 * Scripts/webkitpy/port/ios_device.py: 35 (IOSDevicePort): 36 (IOSDevicePort.default_child_processes): Deleted. 37 * Scripts/webkitpy/port/ios_simulator.py: 38 (IOSSimulatorPort.default_child_processes): Deleted. 39 (IOSSimulatorPort.check_sys_deps): Deleted. 40 * Scripts/webkitpy/port/mac.py: 41 (MacPort.default_child_processes): Accept additional arguments. 42 * Scripts/webkitpy/port/test.py: 43 * Scripts/webkitpy/port/watch_device.py: 44 (WatchDevicePort): 45 (WatchDevicePort.default_child_processes): Deleted. 46 * Scripts/webkitpy/port/watch_simulator.py: 47 (WatchSimulatorPort.default_child_processes): Deleted. 48 (WatchSimulatorPort.check_sys_deps): Deleted. 49 * Scripts/webkitpy/xcode/simulated_device.py: 50 (SimulatedDeviceManager): 51 (SimulatedDeviceManager.device_count_for_type): Count the number of devices 52 available for a specific device type. 53 1 54 2018-12-05 Devin Rousso <drousso@apple.com> 2 55 -
trunk/Tools/Scripts/webkitpy/common/system/platforminfo.py
r237836 r238903 73 73 return self.os_name == 'ios' 74 74 75 def is_watchos(self): 76 return self.os_name == 'watchos' 77 75 78 def is_win(self): 76 79 return self.os_name == 'win' … … 180 183 if sys_platform == 'darwin': 181 184 return 'mac' 182 if sys_platform == 'ios' :185 if sys_platform == 'ios' or sys_platform == 'watchos': 183 186 return 'ios' 184 187 if sys_platform.startswith('linux'): -
trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py
r238749 r238903 105 105 106 106 def _custom_device_for_test(self, test): 107 # FIXME: Use available devices instead of CUSTOM_DEVICE_TYPES https://bugs.webkit.org/show_bug.cgi?id=192161108 107 # FIXME: This is a terrible way to do device-specific expected results https://bugs.webkit.org/show_bug.cgi?id=192162 109 108 for device_type in self._port.CUSTOM_DEVICE_TYPES: … … 207 206 return test_run_results.RunDetails(exit_code=-1) 208 207 209 default_device_tests = []210 211 208 # Look for tests with custom device requirements. 212 custom_device_tests= defaultdict(list)209 test_device_mapping = defaultdict(list) 213 210 for test_file in tests_to_run: 214 custom_device = self._custom_device_for_test(test_file) 215 if custom_device: 216 custom_device_tests[custom_device].append(test_file) 211 test_device_mapping[self._custom_device_for_test(test_file) or self._port.DEFAULT_DEVICE_TYPE].append(test_file) 212 213 # Order device types from most specific to least specific in the hopes that some of the more specific device 214 # types will match the less specific device types. 215 device_type_order = [] 216 types_with_family = [] 217 remaining_types = [] 218 for device_type in test_device_mapping.iterkeys(): 219 if device_type and device_type.hardware_family and device_type.hardware_type: 220 device_type_order.append(device_type) 221 elif device_type and device_type.hardware_family: 222 types_with_family.append(device_type) 217 223 else: 218 default_device_tests.append(test_file) 219 220 if custom_device_tests: 221 for device_type, tests in custom_device_tests.iteritems(): 222 _log.debug('{} tests use device {}'.format(len(tests), device_type)) 223 224 initial_results = None 225 retry_results = None 226 enabled_pixel_tests_in_retry = False 224 remaining_types.append(device_type) 225 device_type_order.extend(types_with_family + remaining_types) 227 226 228 227 needs_http = any((self._is_http_test(test) and not self._needs_web_platform_test(test)) for test in tests_to_run) … … 243 242 self._port.host.filesystem.maybe_make_directory(self._results_directory) 244 243 245 if default_device_tests: 244 initial_results = None 245 retry_results = None 246 enabled_pixel_tests_in_retry = False 247 248 child_processes_option_value = self._options.child_processes 249 250 while device_type_order: 251 device_type = device_type_order[0] 252 tests = test_device_mapping[device_type] 253 del device_type_order[0] 254 255 self._options.child_processes = min(self._port.max_child_processes(device_type=device_type), int(child_processes_option_value or self._port.default_child_processes(device_type=device_type))) 256 246 257 _log.info('') 247 _log.info("Running %s", pluralize(len(tests_to_run), "test")) 258 if not self._options.child_processes: 259 _log.info('Skipping {} because {} is not available'.format(pluralize(len(test_device_mapping[device_type]), 'test'), str(device_type))) 260 _log.info('') 261 continue 262 263 # This loop looks for any less-specific device types which match the current device type 264 index = 0 265 while index < len(device_type_order): 266 if device_type_order[index] == device_type: 267 tests.extend(test_device_mapping[device_type_order[index]]) 268 269 # Remove devices types from device_type_order once tests associated with that type have been claimed. 270 del device_type_order[index] 271 else: 272 index += 1 273 274 _log.info('Running {}{}'.format(pluralize(len(tests), 'test'), ' for {}'.format(str(device_type)) if device_type else '')) 248 275 _log.info('') 249 if not self._set_up_run(tests _to_run):276 if not self._set_up_run(tests, device_type): 250 277 return test_run_results.RunDetails(exit_code=-1) 251 278 252 initial_results, retry_results, enabled_pixel_tests_in_retry = self._run_test_subset(default_device_tests, tests_to_skip) 253 254 # Only use a single worker for custom device classes 255 self._options.child_processes = 1 256 for device_type in custom_device_tests: 257 device_tests = custom_device_tests[device_type] 258 if device_tests: 259 _log.info('') 260 _log.info('Running %s for %s', pluralize(len(device_tests), "test"), device_type) 261 _log.info('') 262 if not self._set_up_run(device_tests, device_type): 263 return test_run_results.RunDetails(exit_code=-1) 264 265 device_initial_results, device_retry_results, device_enabled_pixel_tests_in_retry = self._run_test_subset(device_tests, tests_to_skip) 266 267 initial_results = initial_results.merge(device_initial_results) if initial_results else device_initial_results 268 retry_results = retry_results.merge(device_retry_results) if retry_results else device_retry_results 269 enabled_pixel_tests_in_retry |= device_enabled_pixel_tests_in_retry 279 temp_initial_results, temp_retry_results, temp_enabled_pixel_tests_in_retry = self._run_test_subset(tests, tests_to_skip) 280 initial_results = initial_results.merge(temp_initial_results) if initial_results else temp_initial_results 281 retry_results = retry_results.merge(temp_retry_results) if retry_results else temp_retry_results 282 enabled_pixel_tests_in_retry |= temp_enabled_pixel_tests_in_retry 270 283 271 284 self._runner.stop_servers() 285 272 286 end_time = time.time() 273 287 return self._end_test_run(start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry) … … 299 313 300 314 def _end_test_run(self, start_time, end_time, initial_results, retry_results, enabled_pixel_tests_in_retry): 315 if initial_results is None: 316 _log.error('No results generated') 317 return test_run_results.RunDetails(exit_code=-1) 318 301 319 # Some crash logs can take a long time to be written out so look 302 320 # for new logs after the test run finishes. 303 304 321 _log.debug("looking for new crash logs") 305 322 self._look_for_new_crash_logs(initial_results, start_time) -
trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
r235467 r238903 371 371 def _set_up_derived_options(port, options): 372 372 """Sets the options values that depend on other options values.""" 373 if not options.child_processes:374 options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",375 str(port.default_child_processes()))376 377 373 if not options.configuration: 378 374 options.configuration = port.default_configuration() -
trunk/Tools/Scripts/webkitpy/port/base.py
r238749 r238903 82 82 DEFAULT_ARCHITECTURE = 'x86' 83 83 84 DEFAULT_DEVICE_TYPE = None 84 85 CUSTOM_DEVICE_TYPES = [] 85 86 … … 177 178 return False 178 179 179 def default_child_processes(self ):180 def default_child_processes(self, **kwargs): 180 181 """Return the number of DumpRenderTree instances to use for this port.""" 181 182 return self._executive.cpu_count() 183 184 def max_child_processes(self, **kwargs): 185 """Forbid the user from specifying more than this number of child processes""" 186 return float('inf') 182 187 183 188 def worker_startup_delay_secs(self): -
trunk/Tools/Scripts/webkitpy/port/device_port.py
r238749 r238903 24 24 import traceback 25 25 26 from webkitpy.common.memoized import memoized27 26 from webkitpy.layout_tests.models.test_configuration import TestConfiguration 28 27 from webkitpy.port.darwin import DarwinPort 29 28 from webkitpy.port.simulator_process import SimulatorProcess 30 29 from webkitpy.xcode.device_type import DeviceType 31 from webkitpy.xcode.simulated_device import DeviceRequest 30 from webkitpy.xcode.simulated_device import DeviceRequest, SimulatedDeviceManager 32 31 33 32 … … 38 37 39 38 DEVICE_MANAGER = None 40 DEFAULT_DEVICE_TYPE = None41 39 NO_DEVICE_MANAGER = 'No device manager found for port' 42 40 … … 60 58 return configurations 61 59 62 @memoized63 60 def child_processes(self): 64 61 return int(self.get_option('child_processes')) … … 107 104 raise RuntimeError('Failed to install dylibs at {} on device {}'.format(self._build_path(), device.udid)) 108 105 109 def setup_test_run(self, device_type=None): 110 if not self.DEVICE_MANAGER: 111 raise RuntimeError(self.NO_DEVICE_MANAGER) 112 106 def _device_type_with_version(self, device_type=None): 113 107 device_type = device_type if device_type else self.DEFAULT_DEVICE_TYPE 114 device_type =DeviceType(108 return DeviceType( 115 109 hardware_family=device_type.hardware_family, 116 110 hardware_type=device_type.hardware_type, … … 118 112 software_variant=device_type.software_variant, 119 113 ) 114 115 def default_child_processes(self, device_type=None): 116 if not self.DEVICE_MANAGER: 117 raise RuntimeError(self.NO_DEVICE_MANAGER) 118 if self.DEVICE_MANAGER.INITIALIZED_DEVICES: 119 return len(self.DEVICE_MANAGER.INITIALIZED_DEVICES) 120 return self.DEVICE_MANAGER.device_count_for_type( 121 self._device_type_with_version(device_type), 122 host=self.host, 123 dedicated_simulators=not self.get_option('dedicated_simulators', False), 124 ) 125 126 def max_child_processes(self, device_type=None): 127 if self.DEVICE_MANAGER == SimulatedDeviceManager: 128 return super(DevicePort, self).max_child_processes(device_type=device_type) 129 return self.default_child_processes(device_type=device_type) 130 131 def setup_test_run(self, device_type=None): 132 if not self.DEVICE_MANAGER: 133 raise RuntimeError(self.NO_DEVICE_MANAGER) 134 135 device_type = self._device_type_with_version(device_type) 120 136 _log.debug('\nCreating devices for {}'.format(device_type)) 121 137 … … 126 142 allow_incomplete_match=True, 127 143 ) 128 self.DEVICE_MANAGER.initialize_devices([request] * self.child_processes(), self.host) 144 self.DEVICE_MANAGER.initialize_devices( 145 [request] * self.child_processes(), 146 self.host, 147 layout_test_dir=self.layout_tests_dir(), 148 pin=self.get_option('pin', None), 149 use_nfs=self.get_option('use_nfs', True), 150 reboot=self.get_option('reboot', False), 151 ) 129 152 130 153 if not self.devices(): -
trunk/Tools/Scripts/webkitpy/port/ios_device.py
r238694 r238903 45 45 NO_DEVICE_MANAGER = NO_ON_DEVICE_TESTING 46 46 47 @memoized48 def default_child_processes(self):49 if apple_additions():50 return apple_additions().ios_device_default_child_processes(self)51 return 152 53 47 def _driver_class(self): 54 48 if apple_additions(): … … 93 87 return Version.from_string(self.get_option('version')) 94 88 95 if not apple_additions():89 if not self.DEVICE_MANAGER: 96 90 raise RuntimeError(self.NO_ON_DEVICE_TESTING) 97 91 98 if not self. devices():92 if not self.DEVICE_MANAGER.available_devices(host=self.host): 99 93 raise RuntimeError('No devices are available') 100 94 version = None 101 for device in self.devices(): 95 for device in self.DEVICE_MANAGER.available_devices(host=self.host): 96 if not device.platform.is_ios(): 97 continue 102 98 if not version: 103 99 version = device.platform.os_version -
trunk/Tools/Scripts/webkitpy/port/ios_simulator.py
r238749 r238903 59 59 return IOSSimulatorPort._version_from_name(self._name) if IOSSimulatorPort._version_from_name(self._name) else self.host.platform.xcode_sdk_version('iphonesimulator') 60 60 61 @memoized62 def default_child_processes(self):63 def booted_ios_devices_filter(device):64 if not device.platform_device.is_booted_or_booting():65 return False66 return device.platform_device.device_type in DeviceType(software_variant='iOS', software_version=self.device_version())67 68 if not self.get_option('dedicated_simulators', False):69 num_booted_sims = len(SimulatedDeviceManager.device_by_filter(booted_ios_devices_filter, host=self.host))70 if num_booted_sims:71 return num_booted_sims72 return SimulatedDeviceManager.max_supported_simulators(self.host)73 74 61 def clean_up_test_run(self): 75 62 super(IOSSimulatorPort, self).clean_up_test_run() … … 103 90 return 'ios-simulator' 104 91 105 def check_sys_deps(self):106 target_device_type = DeviceType(software_variant='iOS', software_version=self.device_version())107 for device in SimulatedDeviceManager.available_devices(self.host):108 if device.platform_device.device_type in target_device_type:109 return super(IOSSimulatorPort, self).check_sys_deps()110 _log.error('No Simulated device matching "{}" defined in Xcode iOS SDK'.format(str(target_device_type)))111 return False112 113 92 def reset_preferences(self): 114 93 _log.debug("reset_preferences") -
trunk/Tools/Scripts/webkitpy/port/mac.py
r233831 r238903 189 189 return self._version == 'mavericks' 190 190 191 def default_child_processes(self ):191 def default_child_processes(self, **kwargs): 192 192 default_count = super(MacPort, self).default_child_processes() 193 193 -
trunk/Tools/Scripts/webkitpy/port/test.py
r238749 r238903 419 419 return [self._webkit_baseline_path(d) for d in search_paths[self.name()]] 420 420 421 def default_child_processes(self ):421 def default_child_processes(self, **kwargs): 422 422 return 1 423 423 -
trunk/Tools/Scripts/webkitpy/port/watch_device.py
r238694 r238903 43 43 NO_DEVICE_MANAGER = NO_ON_DEVICE_TESTING 44 44 45 @memoized46 def default_child_processes(self):47 if apple_additions():48 return len(apple_additions().device_for_worker_number_map(self, software_variant='watchOS'))49 return 150 51 45 def _driver_class(self): 52 46 if apple_additions(): … … 91 85 return Version.from_string(self.get_option('version')) 92 86 93 if not apple_additions():87 if not self.DEVICE_MANAGER: 94 88 raise RuntimeError(self.NO_ON_DEVICE_TESTING) 95 89 96 if not apple_additions().device_for_worker_number_map(self, software_variant='watchOS'):90 if not self.DEVICE_MANAGER.available_devices(host=self.host): 97 91 raise RuntimeError('No devices are available') 98 92 version = None 99 for device in self.devices(): 93 for device in self.DEVICE_MANAGER.available_devices(host=self.host): 94 if not device.platform.is_watchos(): 95 continue 100 96 if not version: 101 97 version = device.platform.os_version -
trunk/Tools/Scripts/webkitpy/port/watch_simulator.py
r238749 r238903 72 72 return new_environment 73 73 74 @memoized75 def default_child_processes(self):76 def filter_booted_watchos_devices(device):77 if not device.platform_device.is_booted_or_booting():78 return False79 return device.platform_device.device_type in DeviceType(software_variant='watchOS', software_version=self.device_version())80 81 if not self.get_option('dedicated_simulators', False):82 num_booted_sims = len(SimulatedDeviceManager.device_by_filter(filter_booted_watchos_devices, host=self.host))83 if num_booted_sims:84 return num_booted_sims85 return SimulatedDeviceManager.max_supported_simulators(self.host)86 87 74 def operating_system(self): 88 75 return 'watchos-simulator' 89 90 def check_sys_deps(self):91 target_device_type = DeviceType(software_variant='watchOS', software_version=self.device_version())92 for device in SimulatedDeviceManager.available_devices(self.host):93 if device.platform_device.device_type in target_device_type:94 return super(WatchSimulatorPort, self).check_sys_deps()95 _log.error('No simulated device matching "{}" found in watchOS SDK'.format(str(target_device_type)))96 return False97 76 98 77 def setup_environ_for_server(self, server_name=None): -
trunk/Tools/Scripts/webkitpy/xcode/simulated_device.py
r238749 r238903 326 326 327 327 @staticmethod 328 def device_count_for_type(device_type, host=SystemHost(), use_booted_simulator=True, **kwargs): 329 if not host.platform.is_mac(): 330 return 0 331 332 if SimulatedDeviceManager.device_by_filter(lambda device: device.platform_device.is_booted_or_booting(), host=host) and use_booted_simulator: 333 filter = lambda device: device.platform_device.is_booted_or_booting() and device.platform_device.device_type in device_type 334 return len(SimulatedDeviceManager.device_by_filter(filter, host=host)) 335 336 for name in SimulatedDeviceManager._device_identifier_to_name.itervalues(): 337 if DeviceType.from_string(name) in device_type: 338 return SimulatedDeviceManager.max_supported_simulators(host) 339 return 0 340 341 @staticmethod 328 342 def initialize_devices(requests, host=SystemHost(), name_base='Managed', simulator_ui=True, timeout=SIMULATOR_BOOT_TIMEOUT, **kwargs): 329 343 if SimulatedDeviceManager.INITIALIZED_DEVICES is not None:
Note: See TracChangeset
for help on using the changeset viewer.