Changeset 57381 in webkit


Ignore:
Timestamp:
Apr 9, 2010 4:19:39 PM (14 years ago)
Author:
abarth@webkit.org
Message:

2010-04-09 Adam Barth <​abarth@webkit.org>

Reviewed by Eric Seidel.

new-run-webkit-tests should talk about DumpRenderTree not test_shell
​https://bugs.webkit.org/show_bug.cgi?id=37371

test_shell is some strange Chromium thing.
DumpRenderTree (tm) is the real deal.

  • Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py: Added.
  • Scripts/webkitpy/layout_tests/layout_package/test_expectations.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_shell_thread.py: Removed.
  • Scripts/webkitpy/layout_tests/port/chromium.py:
  • Scripts/webkitpy/layout_tests/port/server_process.py:
  • Scripts/webkitpy/layout_tests/run_webkit_tests.py:
Location:
trunk/WebKitTools
Files:
6 edited
1 moved

Legend:

Unmodified
Added
Removed

  • trunk/WebKitTools/ChangeLog

    r57380 r57381  
     12010-04-09  Adam Barth  <abarth@webkit.org>
     2
     3        Reviewed by Eric Seidel.
     4
     5        new-run-webkit-tests should talk about DumpRenderTree not test_shell
     6        https://bugs.webkit.org/show_bug.cgi?id=37371
     7
     8        test_shell is some strange Chromium thing.
     9        DumpRenderTree (tm) is the real deal.
     10
     11        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py: Added.
     12        * Scripts/webkitpy/layout_tests/layout_package/test_expectations.py:
     13        * Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
     14        * Scripts/webkitpy/layout_tests/layout_package/test_shell_thread.py: Removed.
     15        * Scripts/webkitpy/layout_tests/port/chromium.py:
     16        * Scripts/webkitpy/layout_tests/port/server_process.py:
     17        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
     18
    1192010-04-09  Zoltan Horvath  <zoltan@webkit.org>
    220

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py

    r57380 r57381  
    2828# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    2929
    30 """A Thread object for running the test shell and processing URLs from a
     30"""A Thread object for running DumpRenderTree and processing URLs from a
    3131shared queue.
    3232
    33 Each thread runs a separate instance of the test_shell binary and validates
     33Each thread runs a separate instance of the DumpRenderTree binary and validates
    3434the output.  When there are no more URLs to process in the shared queue, the
    3535thread exits.
    … …  
    4949
    5050_log = logging.getLogger("webkitpy.layout_tests.layout_package."
    51                          "test_shell_thread")
     51                         "dump_render_tree_thread")
    5252
    5353
    … …  
    5555                   output_dir, crash, timeout, test_run_time, actual_checksum,
    5656                   output, error):
    57     """Receives the output from a test_shell process, subjects it to a number
     57    """Receives the output from a DumpRenderTree process, subjects it to a number
    5858    of tests, and returns a list of failure types the test produced.
    5959
    6060    Args:
    6161      port: port-specific hooks
    62       proc: an active test_shell process
     62      proc: an active DumpRenderTree process
    6363      test_info: Object containing the test filename, uri and timeout
    6464      test_types: list of test types to subject the output to
    … …  
    172172    def __init__(self, port, filename_list_queue, result_queue,
    173173                 test_types, test_args, image_path, shell_args, options):
    174         """Initialize all the local state for this test shell thread.
     174        """Initialize all the local state for this DumpRenderTree thread.
    175175
    176176        Args:
    … …  
    183183              against.
    184184          test_args: A TestArguments object to pass to each TestType.
    185           shell_args: Any extra arguments to be passed to test_shell.exe.
     185          shell_args: Any extra arguments to be passed to DumpRenderTree.
    186186          options: A property dictionary as produced by optparse. The
    187187              command-line options should match those expected by
    … …  
    305305                        self._filename_list_queue.get_nowait()
    306306                except Queue.Empty:
    307                     self._kill_test_shell()
     307                    self._kill_dump_render_tree()
    308308                    tests_run_file.close()
    309309                    return
    … …  
    325325            tests_run_file.write(filename + "\n")
    326326            if failures:
    327                 # Check and kill test shell if we need too.
    328                 if len([1 for f in failures if f.should_kill_test_shell()]):
    329                     self._kill_test_shell()
     327                # Check and kill DumpRenderTree if we need too.
     328                if len([1 for f in failures if f.should_kill_dump_render_tree()]):
     329                    self._kill_dump_render_tree()
    330330                    # Reset the batch count since the shell just bounced.
    331331                    batch_count = 0
    … …  
    342342            if batch_size > 0 and batch_count > batch_size:
    343343                # Bounce the shell and reset count.
    344                 self._kill_test_shell()
     344                self._kill_dump_render_tree()
    345345                batch_count = 0
    346346
    … …  
    371371        worker.start()
    372372
    373         # When we're running one test per test_shell process, we can enforce
    374         # a hard timeout. the test_shell watchdog uses 2.5x the timeout
     373        # When we're running one test per DumpRenderTree process, we can enforce
     374        # a hard timeout. the DumpRenderTree watchdog uses 2.5x the timeout
    375375        # We want to be larger than that.
    376376        worker.join(int(test_info.timeout) * 3.0 / 1000.0)
    377377        if worker.isAlive():
    378378            # If join() returned with the thread still running, the
    379             # test_shell.exe is completely hung and there's nothing
     379            # DumpRenderTree is completely hung and there's nothing
    380380            # more we can do with it.  We have to kill all the
    381             # test_shells to free it up. If we're running more than
    382             # one test_shell thread, we'll end up killing the other
    383             # test_shells too, introducing spurious crashes. We accept that
     381            # DumpRenderTrees to free it up. If we're running more than
     382            # one DumpRenderTree thread, we'll end up killing the other
     383            # DumpRenderTrees too, introducing spurious crashes. We accept that
    384384            # tradeoff in order to avoid losing the rest of this thread's
    385385            # results.
    386             _log.error('Test thread hung: killing all test_shells')
     386            _log.error('Test thread hung: killing all DumpRenderTrees')
    387387            worker._driver.stop()
    388388
    … …  
    399399
    400400    def _run_test(self, test_info):
    401         """Run a single test file using a shared test_shell process.
     401        """Run a single test file using a shared DumpRenderTree process.
    402402
    403403        Args:
    … …  
    407407          A list of TestFailure objects describing the error.
    408408        """
    409         self._ensure_test_shell_is_running()
     409        self._ensure_dump_render_tree_is_running()
    410410        # The pixel_hash is used to avoid doing an image dump if the
    411411        # checksums match, so it should be set to a blank value if we
    … …  
    429429        return stats.failures
    430430
    431     def _ensure_test_shell_is_running(self):
    432         """Start the shared test shell, if it's not running.  Not for use when
    433         running tests singly, since those each start a separate test shell in
     431    def _ensure_dump_render_tree_is_running(self):
     432        """Start the shared DumpRenderTree, if it's not running.  Not for use when
     433        running tests singly, since those each start a separate DumpRenderTree in
    434434        their own thread.
    435435        """
    … …  
    438438                self._image_path, self._shell_args)
    439439
    440     def _kill_test_shell(self):
    441         """Kill the test shell process if it's running."""
     440    def _kill_dump_render_tree(self):
     441        """Kill the DumpRenderTree process if it's running."""
    442442        if self._driver:
    443443            self._driver.stop()

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py

    r57173 r57381  
    272272                                IMAGE_PLUS_TEXT: ('image and text mismatch',
    273273                                                  'image and text mismatch'),
    274                                 CRASH: ('test shell crash',
    275                                         'test shell crashes'),
     274                                CRASH: ('DumpRenderTree crash',
     275                                        'DumpRenderTree crashes'),
    276276                                TIMEOUT: ('test timed out', 'tests timed out'),
    277277                                MISSING: ('no expected result found',

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py

    r54328 r57381  
    8080        raise NotImplemented
    8181
    82     def should_kill_test_shell(self):
    83         """Returns True if we should kill the test shell before the next
     82    def should_kill_dump_render_tree(self):
     83        """Returns True if we should kill DumpRenderTree before the next
    8484        test."""
    8585        return False
    … …  
    146146
    147147class FailureTimeout(TestFailure):
    148     """Test timed out.  We also want to restart the test shell if this
     148    """Test timed out.  We also want to restart DumpRenderTree if this
    149149    happens."""
    150150
    … …  
    156156        return "<strong>%s</strong>" % self.message()
    157157
    158     def should_kill_test_shell(self):
     158    def should_kill_dump_render_tree(self):
    159159        return True
    160160
    … …  
    173173                                                             stack)
    174174
    175     def should_kill_test_shell(self):
     175    def should_kill_dump_render_tree(self):
    176176        return True
    177177

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py

    r57256 r57381  
    7777    def check_build(self, needs_http):
    7878        result = True
    79         test_shell_binary_path = self._path_to_driver()
    80         result = check_file_exists(test_shell_binary_path,
     79        dump_render_tree_binary_path = self._path_to_driver()
     80        result = check_file_exists(dump_render_tree_binary_path,
    8181                                   'test driver')
    8282        if result:
    … …  
    9898
    9999    def check_sys_deps(self, needs_http):
    100         test_shell_binary_path = self._path_to_driver()
    101         proc = subprocess.Popen([test_shell_binary_path,
     100        dump_render_tree_binary_path = self._path_to_driver()
     101        proc = subprocess.Popen([dump_render_tree_binary_path,
    102102                                '--check-layout-test-sys-deps'])
    103103        if proc.wait():
    … …  
    135135    def setup_test_run(self):
    136136        # Delete the disk cache if any to ensure a clean test run.
    137         test_shell_binary_path = self._path_to_driver()
    138         cachedir = os.path.split(test_shell_binary_path)[0]
     137        dump_render_tree_binary_path = self._path_to_driver()
     138        cachedir = os.path.split(dump_render_tree_binary_path)[0]
    139139        cachedir = os.path.join(cachedir, "cache")
    140140        if os.path.exists(cachedir):
    … …  
    214214                    release_mtime > debug_mtime and configuration == 'Debug'):
    215215                    _log.warning('You are not running the most '
    216                                  'recent test_shell binary. You need to '
     216                                 'recent DumpRenderTree binary. You need to '
    217217                                 'pass --debug or not to select between '
    218218                                 'Debug and Release.')
    … …  
    257257
    258258        # We need to pass close_fds=True to work around Python bug #2320
    259         # (otherwise we can hang when we kill test_shell when we are running
     259        # (otherwise we can hang when we kill DumpRenderTree when we are running
    260260        # multiple threads). See http://bugs.python.org/issue2320 .
    261261        # Note that close_fds isn't supported on Windows, but this bug only
    … …  
    295295                # This is hex code 0xc000001d, which is used for abrupt
    296296                # termination. This happens if we hit ctrl+c from the prompt
    297                 # and we happen to be waiting on the test_shell.
     297                # and we happen to be waiting on the DumpRenderTree.
    298298                # sdoyon: Not sure for which OS and in what circumstances the
    299299                # above code is valid. What works for me under Linux to detect
    … …  
    337337                # Closing stdin/stdout/stderr hangs sometimes on OS X,
    338338                # (see __init__(), above), and anyway we don't want to hang
    339                 # the harness if test_shell is buggy, so we wait a couple
    340                 # seconds to give test_shell a chance to clean up, but then
     339                # the harness if DumpRenderTree is buggy, so we wait a couple
     340                # seconds to give DumpRenderTree a chance to clean up, but then
    341341                # force-kill the process if necessary.
    342342                KILL_TIMEOUT = 3.0

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py

    r56647 r57381  
    8787            # This is hex code 0xc000001d, which is used for abrupt
    8888            # termination. This happens if we hit ctrl+c from the prompt
    89             # and we happen to be waiting on the test_shell.
     89            # and we happen to be waiting on the DumpRenderTree.
    9090            # sdoyon: Not sure for which OS and in what circumstances the
    9191            # above code is valid. What works for me under Linux to detect
    … …  
    206206            # Closing stdin/stdout/stderr hangs sometimes on OS X,
    207207            # (see restart(), above), and anyway we don't want to hang
    208             # the harness if test_shell is buggy, so we wait a couple
    209             # seconds to give test_shell a chance to clean up, but then
     208            # the harness if DumpRenderTree is buggy, so we wait a couple
     209            # seconds to give DumpRenderTree a chance to clean up, but then
    210210            # force-kill the process if necessary.
    211211            KILL_TIMEOUT = 3.0

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

    r57265 r57381  
    2828# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    2929
    30 """Run layout tests using the test_shell.
     30"""Run layout tests using DumpRenderTree.
    3131
    3232This is a port of the existing webkit test script run-webkit-tests.
    … …  
    6565from layout_package import metered_stream
    6666from layout_package import test_failures
    67 from layout_package import test_shell_thread
     67from layout_package import dump_render_tree_thread
    6868from layout_package import test_files
    6969from test_types import fuzzy_image_diff
    … …  
    165165    # The per-test timeout in milliseconds, if no --time-out-ms option was
    166166    # given to run_webkit_tests. This should correspond to the default timeout
    167     # in test_shell.exe.
     167    # in DumpRenderTree.
    168168    DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
    169169
    … …  
    476476        return filename_queue
    477477
    478     def _get_test_shell_args(self, index):
    479         """Returns the tuple of arguments for tests and for test_shell."""
     478    def _get_dump_render_tree_args(self, index):
     479        """Returns the tuple of arguments for tests and for DumpRenderTree."""
    480480        shell_args = []
    481481        test_args = test_type_base.TestArguments()
    … …  
    505505        return False
    506506
    507     def _instantiate_test_shell_threads(self, test_files, result_summary):
     507    def _instantiate_dump_render_tree_threads(self, test_files, result_summary):
    508508        """Instantitates and starts the TestShellThread(s).
    509509
    … …  
    515515        # Instantiate TestShellThreads and start them.
    516516        threads = []
    517         for i in xrange(int(self._options.num_test_shells)):
     517        for i in xrange(int(self._options.num_dump_render_trees)):
    518518            # Create separate TestTypes instances for each thread.
    519519            test_types = []
    … …  
    522522                                    self._options.results_directory))
    523523
    524             test_args, png_path, shell_args = self._get_test_shell_args(i)
    525             thread = test_shell_thread.TestShellThread(self._port,
    526                                                        filename_queue,
    527                                                        self._result_queue,
    528                                                        test_types,
    529                                                        test_args,
    530                                                        png_path,
    531                                                        shell_args,
    532                                                        self._options)
     524            test_args, png_path, shell_args = self._get_dump_render_tree_args(i)
     525            thread = dump_render_tree_thread.TestShellThread(self._port,
     526                                                             filename_queue,
     527                                                             self._result_queue,
     528                                                             test_types,
     529                                                             test_args,
     530                                                             png_path,
     531                                                             shell_args,
     532                                                             self._options)
    533533            if self._is_single_threaded():
    534534                thread.run_in_main_thread(self, result_summary)
    … …  
    541541    def _is_single_threaded(self):
    542542        """Returns whether we should run all the tests in the main thread."""
    543         return int(self._options.num_test_shells) == 1
     543        return int(self._options.num_dump_render_trees) == 1
    544544
    545545    def _run_tests(self, file_list, result_summary):
    … …  
    557557            result_summary: summary object to populate with the results
    558558        """
    559         self._meter.update('Starting test shells ...')
    560         threads = self._instantiate_test_shell_threads(file_list,
    561                                                        result_summary)
     559        self._meter.update('Starting DumpRenderTrees ...')
     560        threads = self._instantiate_dump_render_tree_threads(file_list,
     561                                                             result_summary)
    562562
    563563        # Wait for the threads to finish and collect test failures.
    … …  
    684684
    685685        # Write the summary to disk (results.html) and maybe open the
    686         # test_shell to this file.
     686        # DumpRenderTree to this file.
    687687        wrote_results = self._write_results_html_file(result_summary)
    688688        if not self._options.noshow_results and wrote_results:
    … …  
    958958            cuml_time += t['total_time']
    959959        write("   %6.2f cumulative, %6.2f optimal" %
    960               (cuml_time, cuml_time / int(self._options.num_test_shells)))
     960              (cuml_time, cuml_time / int(self._options.num_dump_render_trees)))
    961961        write("")
    962962
    … …  
    971971          write: A callback to write info to (e.g., a LoggingWriter) or
    972972              sys.stdout.write.
    973           individual_test_timings: List of test_shell_thread.TestStats for all
     973          individual_test_timings: List of dump_render_tree_thread.TestStats for all
    974974              tests.
    975975        """
    976976        test_types = individual_test_timings[0].time_for_diffs.keys()
    977         times_for_test_shell = []
     977        times_for_dump_render_tree = []
    978978        times_for_diff_processing = []
    979979        times_per_test_type = {}
    … …  
    982982
    983983        for test_stats in individual_test_timings:
    984             times_for_test_shell.append(test_stats.test_run_time)
     984            times_for_dump_render_tree.append(test_stats.test_run_time)
    985985            times_for_diff_processing.append(
    986986                test_stats.total_time_for_all_diffs)
    … …  
    991991
    992992        self._print_statistics_for_test_timings(write,
    993             "PER TEST TIME IN TESTSHELL (seconds):", times_for_test_shell)
     993            "PER TEST TIME IN TESTSHELL (seconds):", times_for_dump_render_tree)
    994994        self._print_statistics_for_test_timings(write,
    995995            "PER TEST DIFF PROCESSING TIMES (seconds):",
    … …  
    10061006          write: A callback to write info to (e.g., a LoggingWriter) or
    10071007              sys.stdout.write.
    1008           individual_test_timings: List of test_shell_thread.TestStats for all
     1008          individual_test_timings: List of dump_render_tree_thread.TestStats for all
    10091009              tests.
    10101010          result_summary: summary object for test run
    10111011        """
    1012         # Reverse-sort by the time spent in test_shell.
     1012        # Reverse-sort by the time spent in DumpRenderTree.
    10131013        individual_test_timings.sort(lambda a, b:
    10141014            cmp(b.test_run_time, a.test_run_time))
    … …  
    13311331
    13321332    def _show_results_html_file(self):
    1333         """Launches the test shell open to the results.html page."""
     1333        """Shows the results.html page."""
    13341334        results_filename = os.path.join(self._options.results_directory,
    13351335                                        "results.html")
    … …  
    14221422                              ignore_errors=True)
    14231423
    1424     if not options.num_test_shells:
     1424    if not options.num_dump_render_trees:
    14251425        # TODO(ojan): Investigate perf/flakiness impact of using numcores + 1.
    1426         options.num_test_shells = port_obj.num_cores()
     1426        options.num_dump_render_trees = port_obj.num_cores()
    14271427
    14281428    write = create_logging_writer(options, 'config')
    1429     write("Running %s test_shells in parallel" % options.num_test_shells)
     1429    write("Running %s DumpRenderTrees in parallel" % options.num_dump_render_trees)
    14301430
    14311431    if not options.time_out_ms:
    … …  
    15441544                                  "whatever's already there.")
    15451545    option_parser.add_option("", "--noshow-results", action="store_true",
    1546                              default=False, help="don't launch the test_shell"
     1546                             default=False, help="don't launch DumpRenderTree"
    15471547                             " with results after the tests are done")
    15481548    option_parser.add_option("", "--full-results-html", action="store_true",
    … …  
    15601560                             help="Run all tests, even those marked SKIP "
    15611561                                  "in the test list")
    1562     option_parser.add_option("", "--num-test-shells",
    1563                              help="Number of testshells to run in parallel.")
     1562    option_parser.add_option("", "--num-dump_render_trees",
     1563                             help="Number of DumpRenderTrees to run in parallel.")
    15641564    option_parser.add_option("", "--use-apache", action="store_true",
    15651565                             default=False,
    … …  
    15691569    option_parser.add_option("", "--run-singly", action="store_true",
    15701570                             default=False,
    1571                              help="run a separate test_shell for each test")
     1571                             help="run a separate DumpRenderTree for each test")
    15721572    option_parser.add_option("", "--num-slow-tests-to-log", default=50,
    15731573                             help="Number of slow tests whose timings "
    … …  
    15981598    option_parser.add_option("", "--startup-dialog", action="store_true",
    15991599                             default=False,
    1600                              help="create a dialog on test_shell.exe startup")
     1600                             help="create a dialog on DumpRenderTree startup")
    16011601    option_parser.add_option("", "--gp-fault-error-box", action="store_true",
    16021602                             default=False,
    … …  
    16041604    option_parser.add_option("", "--wrapper",
    16051605                             help="wrapper command to insert before "
    1606                                   "invocations of test_shell; option is split "
     1606                                  "invocations of DumpRenderTree; option is split "
    16071607                                  "on whitespace before running. (Example: "
    16081608                                  "--wrapper='valgrind --smc-check=all')")
    … …  
    16291629                             default=None,
    16301630                             help=("Run a the tests in batches (n), after "
    1631                                    "every n tests, the test shell is "
     1631                                   "every n tests, DumpRenderTree is "
    16321632                                   "relaunched."))
    16331633    option_parser.add_option("", "--builder-name",
Note: See TracChangeset for help on using the changeset viewer.