Changeset 72458 in webkit


Ignore:
Timestamp:
Nov 19, 2010 5:49:50 PM (13 years ago)
Author:
dpranke@chromium.org
Message:

2010-11-19 Dirk Pranke <dpranke@chromium.org>

Reviewed by Tony Chang.

nrwt multiprocessing - add 'worker number' concept, move stuff to worker thread

Add the 'worker number' and 'worker name' concepts to the
TestShellThread objects, and move test_types and test_args from
the TestRunner to the TestShellThread.

https://bugs.webkit.org/show_bug.cgi?id=49768

  • Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
  • Scripts/webkitpy/layout_tests/run_webkit_tests.py:
Location:
trunk/WebKitTools
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/WebKitTools/ChangeLog

    r72457 r72458  
     12010-11-19  Dirk Pranke  <dpranke@chromium.org>
     2
     3        Reviewed by Tony Chang.
     4
     5        nrwt multiprocessing - add 'worker number' concept, move stuff to worker thread
     6
     7        Add the 'worker number' and 'worker name' concepts to the
     8        TestShellThread objects, and move test_types and test_args from
     9        the TestRunner to the TestShellThread.
     10
     11        https://bugs.webkit.org/show_bug.cgi?id=49768
     12
     13        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
     14        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
     15
    1162010-11-19  Dirk Pranke  <dpranke@chromium.org>
    217
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py

    r72457 r72458  
    5050import time
    5151
     52
     53from webkitpy.layout_tests.test_types import image_diff
     54from webkitpy.layout_tests.test_types import test_type_base
     55from webkitpy.layout_tests.test_types import text_diff
     56
    5257import test_failures
    5358import test_output
     
    6570
    6671def _process_output(port, options, test_input, test_types, test_args,
    67                     test_output):
     72                    test_output, worker_name):
    6873    """Receives the output from a DumpRenderTree process, subjects it to a
    6974    number of tests, and returns a list of failure types the test produced.
     
    7782      test_args: arguments to be passed to each test
    7883      test_output: a TestOutput object containing the output of the test
     84      worker_name: worker name for logging
    7985
    8086    Returns: a TestResult object
     
    8793        failures.append(test_failures.FailureTimeout())
    8894
     95    test_name = port.relative_test_filename(test_input.filename)
    8996    if test_output.crash:
    90         _log.debug("Stacktrace for %s:\n%s" % (test_input.filename,
    91                                                test_output.error))
    92         # Strip off "file://" since RelativeTestFilename expects
    93         # filesystem paths.
    94         filename = os.path.join(options.results_directory,
    95                                 port.relative_test_filename(
    96                                 test_input.filename))
     97        _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
     98                                                  test_output.error))
     99        filename = os.path.join(options.results_directory, test_name)
    97100        filename = os.path.splitext(filename)[0] + "-stack.txt"
    98101        port.maybe_make_directory(os.path.split(filename)[0])
     
    100103            file.write(test_output.error)
    101104    elif test_output.error:
    102         _log.debug("Previous test output stderr lines:\n%s" % test_output.error)
     105        _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
     106                                                       test_output.error))
    103107
    104108    expected_test_output = _expected_test_output(port, test_input.filename)
     
    144148
    145149
    146 def _run_single_test(port, options, test_input, test_types, test_args, driver):
     150def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
    147151    # FIXME: Pull this into TestShellThread._run().
    148152
     
    158162    test_output = driver.run_test(uri, test_input.timeout, image_hash_to_driver)
    159163    return _process_output(port, options, test_input, test_types, test_args,
    160                            test_output)
     164                           test_output, worker_name)
    161165
    162166
     
    164168    """Thread wrapper for running a single test file."""
    165169
    166     def __init__(self, port, options, test_input, test_types, test_args):
     170    def __init__(self, port, options, worker_number, worker_name,
     171                 test_input, test_types, test_args):
    167172        """
    168173        Args:
    169174          port: object implementing port-specific hooks
    170175          options: command line argument object from optparse
     176          worker_number: worker number for tests
     177              (FIXME: this should be passed to port.create_driver()).
     178          worker_name: for logging
    171179          test_input: Object containing the test filename and timeout
    172180          test_types: A list of TestType objects to run the test output
     
    182190        self._test_args = test_args
    183191        self._driver = None
     192        self._worker_number = worker_number
     193        self._name = worker_name
    184194
    185195    def run(self):
     
    194204        self._test_result = _run_single_test(self._port, self._options,
    195205                                             self._test_input, self._test_types,
    196                                              self._test_args, self._driver)
     206                                             self._test_args, self._driver,
     207                                             self._name)
    197208        self._driver.stop()
    198209
     
    237248
    238249class TestShellThread(WatchableThread):
    239     def __init__(self, port, options, filename_list_queue, result_queue,
    240                  test_types, test_args):
     250    def __init__(self, port, options, worker_number,
     251                 filename_list_queue, result_queue):
    241252        """Initialize all the local state for this DumpRenderTree thread.
    242253
     
    244255          port: interface to port-specific hooks
    245256          options: command line options argument from optparse
     257          worker_number: identifier for a particular worker thread.
    246258          filename_list_queue: A thread safe Queue class that contains lists
    247259              of tuples of (filename, uri) pairs.
    248260          result_queue: A thread safe Queue class that will contain
    249261              serialized TestResult objects.
    250           test_types: A list of TestType objects to run the test output
    251               against.
    252           test_args: A TestArguments object to pass to each TestType.
    253262        """
    254263        WatchableThread.__init__(self)
    255264        self._port = port
    256265        self._options = options
     266        self._worker_number = worker_number
     267        self._name = 'worker-%d' % worker_number
    257268        self._filename_list_queue = filename_list_queue
    258269        self._result_queue = result_queue
    259270        self._filename_list = []
    260         self._test_types = test_types
    261         self._test_args = test_args
    262271        self._driver = None
    263272        self._test_group_timing_stats = {}
     
    270279        self._http_lock_wait_end = 0
    271280
     281        self._test_types = []
     282        for cls in self._get_test_type_classes():
     283            self._test_types.append(cls(self._port,
     284                                        self._options.results_directory))
     285        self._test_args = self._get_test_args(worker_number)
     286
    272287        # Current group of tests we're running.
    273288        self._current_group = None
     
    276291        # Time at which we started running tests from self._current_group.
    277292        self._current_group_start_time = None
     293
     294    def _get_test_args(self, worker_number):
     295        """Returns the tuple of arguments for tests and for DumpRenderTree."""
     296        test_args = test_type_base.TestArguments()
     297        test_args.png_path = None
     298        if self._options.pixel_tests:
     299            png_path = os.path.join(self._options.results_directory,
     300                                    "png_result%s.png" %
     301                                    self._worker_number)
     302            test_args.png_path = png_path
     303        test_args.new_baseline = self._options.new_baseline
     304        test_args.reset_results = self._options.reset_results
     305
     306        return test_args
     307
     308    def _get_test_type_classes(self):
     309        classes = [text_diff.TestTextDiff]
     310        if self._options.pixel_tests:
     311            classes.append(image_diff.ImageDiff)
     312        return classes
    278313
    279314    def get_test_group_timing_stats(self):
     
    447482        worker = SingleTestThread(self._port,
    448483                                  self._options,
     484                                  self._worker_number,
     485                                  self._name,
    449486                                  test_input,
    450487                                  self._test_types,
     
    496533        test_result = _run_single_test(self._port, self._options, test_input,
    497534                                       self._test_types, self._test_args,
    498                                        self._driver)
     535                                       self._driver, self._name)
    499536        self._test_results.append(test_result)
    500537        return test_result
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

    r72339 r72458  
    7373from layout_package import test_results
    7474from layout_package import test_results_uploader
    75 from test_types import image_diff
    76 from test_types import text_diff
    77 from test_types import test_type_base
    7875
    7976from webkitpy.common.system import user
     
    254251        # self._websocket_secure_server = websocket_server.PyWebSocket(
    255252        #        options.results_directory, use_tls=True, port=9323)
    256 
    257         # a list of TestType objects
    258         self._test_types = [text_diff.TestTextDiff]
    259         if options.pixel_tests:
    260             self._test_types.append(image_diff.ImageDiff)
    261253
    262254        # a set of test files, and the same tests as a list
     
    560552        return filename_queue
    561553
    562     def _get_test_args(self, index):
    563         """Returns the tuple of arguments for tests and for DumpRenderTree."""
    564         test_args = test_type_base.TestArguments()
    565         test_args.png_path = None
    566         if self._options.pixel_tests:
    567             png_path = os.path.join(self._options.results_directory,
    568                                     "png_result%s.png" % index)
    569             test_args.png_path = png_path
    570         test_args.new_baseline = self._options.new_baseline
    571         test_args.reset_results = self._options.reset_results
    572 
    573         return test_args
    574 
    575554    def _contains_tests(self, subdir):
    576555        for test_file in self._test_files:
     
    590569        # Instantiate TestShellThreads and start them.
    591570        threads = []
    592         for i in xrange(int(self._options.child_processes)):
    593             # Create separate TestTypes instances for each thread.
    594             test_types = []
    595             for test_type in self._test_types:
    596                 test_types.append(test_type(self._port,
    597                                     self._options.results_directory))
    598 
    599             test_args = self._get_test_args(i)
     571        for worker_number in xrange(int(self._options.child_processes)):
    600572            thread = dump_render_tree_thread.TestShellThread(self._port,
    601                 self._options, filename_queue, self._result_queue,
    602                 test_types, test_args)
     573                self._options, worker_number,
     574                filename_queue, self._result_queue)
    603575            if self._is_single_threaded():
    604576                thread.run_in_main_thread(self, result_summary)
Note: See TracChangeset for help on using the changeset viewer.