Changeset 121807 in webkit


Ignore:
Timestamp:
Jul 3, 2012 4:11:18 PM (12 years ago)
Author:
dpranke@chromium.org
Message:

nrwt: moving child process logging code into manager_worker_broker
​https://bugs.webkit.org/show_bug.cgi?id=90408

Reviewed by Ojan Vafai.

Users of manager_worker_broker should not have to be aware of
whether they're in the same process or different processes and
configure logging themselves; mwb should hide this complexity.
We can't quite do this completely/correctly yet, since the
manager expects to get a list of messages to log, but this
change fixes the worker side of it, at least.

This is just moving code around, there is no new functionality
and this should be covered by existing tests.

  • Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py:

(AbstractWorker.init):
(_WorkerConnection.init):
(_WorkerConnection.post_message):
(_WorkerConnection):
(_WorkerConnection.set_up_logging):
(_WorkerConnection.clean_up_logging):
(_InlineWorkerConnection.run):
(_MultiProcessWorkerConnection.run):
(_WorkerLogHandler):
(_WorkerLogHandler.init):
(_WorkerLogHandler.emit):

  • Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py:

(_TestWorker.run):
(_TestsMixin.handle_done):

  • Scripts/webkitpy/layout_tests/controllers/worker.py:

(Worker.init):
(Worker.run):
(Worker._run_test):
(Worker.cleanup):
(Worker.run_single_test):

Location:
trunk/Tools
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r121802 r121807  
     12012-07-03  Dirk Pranke  <dpranke@chromium.org>
     2
     3        nrwt: moving child process logging code into manager_worker_broker
     4        https://bugs.webkit.org/show_bug.cgi?id=90408
     5
     6        Reviewed by Ojan Vafai.
     7
     8        Users of manager_worker_broker should not have to be aware of
     9        whether they're in the same process or different processes and
     10        configure logging themselves; mwb should hide this complexity.
     11        We can't quite do this completely/correctly yet, since the
     12        manager expects to get a list of messages to log, but this
     13        change fixes the worker side of it, at least.
     14
     15        This is just moving code around, there is no new functionality
     16        and this should be covered by existing tests.
     17
     18        * Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py:
     19        (AbstractWorker.__init__):
     20        (_WorkerConnection.__init__):
     21        (_WorkerConnection.post_message):
     22        (_WorkerConnection):
     23        (_WorkerConnection.set_up_logging):
     24        (_WorkerConnection.clean_up_logging):
     25        (_InlineWorkerConnection.run):
     26        (_MultiProcessWorkerConnection.run):
     27        (_WorkerLogHandler):
     28        (_WorkerLogHandler.__init__):
     29        (_WorkerLogHandler.emit):
     30        * Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py:
     31        (_TestWorker.run):
     32        (_TestsMixin.handle_done):
     33        * Scripts/webkitpy/layout_tests/controllers/worker.py:
     34        (Worker.__init__):
     35        (Worker.run):
     36        (Worker._run_test):
     37        (Worker.cleanup):
     38        (Worker.run_single_test):
     39
    1402012-07-03  Tony Chang  <tony@chromium.org>
    241

  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker.py

    r121509 r121807  
    7070import multiprocessing
    7171import optparse
     72import os
    7273import Queue
    7374import sys
    … …  
    7677
    7778from webkitpy.common.system import stack_utils
     79from webkitpy.layout_tests.views import metered_stream
    7880
    7981
    … …  
    278280        self._done = False
    279281        self._canceled = False
     282        self._options = optparse.Values({'verbose': False})
    280283
    281284    def name(self):
    … …  
    362365        self._client = worker_factory(self, worker_number)
    363366        self._host = host
     367        self._log_messages = []
     368        self._logger = None
     369        self._log_handler = None
    364370        _BrokerConnection.__init__(self, broker, self._client, ANY_WORKER_TOPIC, MANAGER_TOPIC)
    365371
    … …  
    378384    def yield_to_broker(self):
    379385        pass
     386
     387    def post_message(self, *args):
     388        # FIXME: This is a hack until we can remove the log_messages arg from the manager.
     389        if args[0] in ('finished_test', 'done'):
     390            log_messages = self._log_messages
     391            self._log_messages = []
     392            args = args + tuple([log_messages])
     393        super(_WorkerConnection, self).post_message(*args)
     394
     395    def set_up_logging(self):
     396        self._logger = logging.root
     397        # The unix multiprocessing implementation clones the MeteredStream log handler
     398        # into the child process, so we need to remove it to avoid duplicate logging.
     399        for h in self._logger.handlers:
     400            # log handlers don't have names until python 2.7.
     401            if getattr(h, 'name', '') == metered_stream.LOG_HANDLER_NAME:
     402                self._logger.removeHandler(h)
     403                break
     404        self._logger.setLevel(logging.DEBUG if self._client._options.verbose else logging.INFO)
     405        self._log_handler = _WorkerLogHandler(self)
     406        self._logger.addHandler(self._log_handler)
     407
     408    def clean_up_logging(self):
     409        if self._log_handler and self._logger:
     410            self._logger.removeHandler(self._log_handler)
     411        self._log_handler = None
     412        self._logger = None
    380413
    381414
    … …  
    398431        self._alive = True
    399432        try:
    400             self._client.run(self._host, set_up_logging=False)
     433            self._client.run(self._host)
    401434        finally:
    402435            self._alive = False
    … …  
    440473
    441474    def run(self):
    442         # FIXME: set_up_logging shouldn't be needed.
    443         self._client.run(self._host, set_up_logging=True)
     475        self.set_up_logging()
     476        try:
     477            self._client.run(self._host)
     478        finally:
     479            self.clean_up_logging()
     480
     481
     482class _WorkerLogHandler(logging.Handler):
     483    def __init__(self, worker):
     484        logging.Handler.__init__(self)
     485        self._worker = worker
     486        self._pid = os.getpid()
     487
     488    def emit(self, record):
     489        self._worker._log_messages.append(record)

  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager_worker_broker_unittest.py

    r121509 r121807  
    7070        self._worker_connection.post_message('test', 2, 'hi, ' + self._thing_to_greet)
    7171
    72     def run(self, host, set_up_logging):
     72    def run(self, host):
    7373        if self._starting_queue:
    7474            self._starting_queue.put('')
    … …  
    103103        return self._done
    104104
    105     def handle_done(self, src):
     105    def handle_done(self, src, log_messages):
    106106        self._done = True
    107107

  • trunk/Tools/Scripts/webkitpy/layout_tests/controllers/worker.py

    r121509 r121807  
    3030
    3131import logging
    32 import os
    33 import sys
    3432import threading
    3533import time
    … …  
    4038from webkitpy.layout_tests.models import test_expectations
    4139from webkitpy.layout_tests.models import test_results
    42 from webkitpy.layout_tests.views import metered_stream
    4340
    4441
    … …  
    5855        self._tests_run_file = None
    5956        self._tests_run_filename = None
    60         self._log_messages = []
    61         self._logger = None
    62         self._log_handler = None
    6357
    6458    def __del__(self):
    … …  
    7771        self._tests_run_file = self._filesystem.open_text_file_for_writing(tests_run_filename)
    7872
    79     def _set_up_logging(self):
    80         self._logger = logging.getLogger()
    81 
    82         # The unix multiprocessing implementation clones the MeteredStream log handler
    83         # into the child process, so we need to remove it to avoid duplicate logging.
    84         for h in self._logger.handlers:
    85             # log handlers don't have names until python 2.7.
    86             if getattr(h, 'name', '') == metered_stream.LOG_HANDLER_NAME:
    87                 self._logger.removeHandler(h)
    88                 break
    89 
    90         self._logger.setLevel(logging.DEBUG if self._options.verbose else logging.INFO)
    91         self._log_handler = _WorkerLogHandler(self)
    92         self._logger.addHandler(self._log_handler)
    93 
    94     def run(self, host, set_up_logging):
     73    def run(self, host):
    9574        if not host:
    9675            host = Host()
    97 
    98         # FIXME: this should move into manager_worker_broker.py.
    99         if set_up_logging:
    100             self._set_up_logging()
    10176
    10277        options = self._options
    … …  
    11186            _log.debug("%s exiting" % self._name)
    11287            self.cleanup()
    113             self._worker_connection.post_message('done', self._log_messages)
     88            self._worker_connection.post_message('done')
    11489
    11590    def handle_test_list(self, src, list_name, test_list):
    … …  
    148123
    149124        elapsed_time = time.time() - start
    150         log_messages = self._log_messages
    151         self._log_messages = []
    152         self._worker_connection.post_message('finished_test', result, elapsed_time, log_messages)
     125        self._worker_connection.post_message('finished_test', result, elapsed_time)
    153126
    154127        self.clean_up_after_test(test_input, result)
    … …  
    160133            self._tests_run_file.close()
    161134            self._tests_run_file = None
    162         if self._log_handler and self._logger:
    163             self._logger.removeHandler(self._log_handler)
    164         self._log_handler = None
    165         self._logger = None
    166135
    167136    def timeout(self, test_input):
    … …  
    280249        return single_test_runner.run_single_test(self._port, self._options,
    281250            test_input, driver, self._name)
    282 
    283 
    284 class _WorkerLogHandler(logging.Handler):
    285     def __init__(self, worker):
    286         logging.Handler.__init__(self)
    287         self._worker = worker
    288         self._pid = os.getpid()
    289 
    290     def emit(self, record):
    291         self._worker._log_messages.append(record)
Note: See TracChangeset for help on using the changeset viewer.