Changeset 59631 in webkit


Ignore:
Timestamp:
May 17, 2010 5:15:16 PM (14 years ago)
Author:
eric@webkit.org
Message:

2010-05-17 Eric Seidel <eric@webkit.org>

Reviewed by Adam Barth.

Attempt to make new-run-webkit-tests --help more sane
https://bugs.webkit.org/show_bug.cgi?id=37836

  • Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
    • Add a FIXME about options.singly and options.batch_size being different.
  • Scripts/webkitpy/layout_tests/run_webkit_tests.py:
    • Add support for hidden options.
    • Add option groupings to attempt to simplify --help.
    • Fix a bunch of option helps to start with a capitalized verb.
    • Hide a bunch of options which make no sense to users.
    • Sort options in --help.
  • Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
    • Add tests for option sorting.
Location:
trunk/WebKitTools
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/WebKitTools/ChangeLog

    r59623 r59631  
     12010-05-17  Eric Seidel  <eric@webkit.org>
     2
     3        Reviewed by Adam Barth.
     4
     5        Attempt to make new-run-webkit-tests --help more sane
     6        https://bugs.webkit.org/show_bug.cgi?id=37836
     7
     8        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
     9         - Add a FIXME about options.singly and options.batch_size being different.
     10        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
     11         - Add support for hidden options.
     12         - Add option groupings to attempt to simplify --help.
     13         - Fix a bunch of option helps to start with a capitalized verb.
     14         - Hide a bunch of options which make no sense to users.
     15         - Sort options in --help.
     16        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
     17         - Add tests for option sorting.
     18
    1192010-05-17  Robert Hogan  <robert@webkit.org>
    220
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py

    r58473 r59631  
    325325            batch_count += 1
    326326            self._num_tests += 1
     327            # FIXME: options.run_singly and options.batch_size=1 should use the same code path!
    327328            if self._options.run_singly:
    328329                result = self._run_test_singly(test_info)
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

    r59108 r59631  
    13261326            return 0
    13271327
    1328     if options.clobber_old_results:
     1328    if options.reset_results:
    13291329        # Just clobber the actual test results directories since the other
    13301330        # files in the results directory are explicitly used for cross-run
     
    14361436
    14371437
     1438class HiddenOptionGroup(optparse.OptionGroup):
     1439    def __init__(self, parser):
     1440        optparse.OptionGroup.__init__(self, parser, title=None)
     1441
     1442    def format_help(self, formatter):
     1443        return ""
     1444
     1445
    14381446def _compat_shim_callback(option, opt_str, value, parser):
     1447    """Logs that an option is being ignored."""
    14391448    print "Ignoring unsupported option: %s" % opt_str
    14401449
    14411450
    14421451def _compat_shim_option(option_name, **kwargs):
     1452    """returns an optparse Option which simply logs that it's being ignored."""
    14431453    return optparse.make_option(option_name, action="callback",
    14441454        callback=_compat_shim_callback,
     
    14461456
    14471457
     1458def _deprecated_shim_callback(option, opt_str, value, parser, new_option):
     1459    """Logs that the option is deprecated and then calls new_option.process"""
     1460    print "%s is DEPRECATED please use %s" % (opt_str, new_option)
     1461    # We could time.sleep here or some other discouraging measure.
     1462    new_option.process(opt_str, value, parser.values, parser)
     1463
     1464
     1465def _deprecated_option(parser, option_name, long_name=None, new_option_name=None):
     1466    """Returns an option which maps from and old new to a new name and prints
     1467    a deprecation warning when called."""
     1468    new_option = parser.get_option(new_option_name)
     1469    option_names = [option_name]
     1470    if long_name:
     1471        option_names.append(long_name)
     1472    return optparse.make_option(
     1473        action="callback",
     1474        callback=_deprecated_shim_callback,
     1475        callback_args=(new_option,),
     1476        nargs=new_option.nargs,
     1477        help="DEPRECATED, use %s" % new_option,
     1478        *option_names)
     1479
     1480
     1481def _split_option_name(option_name):
     1482    """Returns the name of the option, ignoring any leading -, --, or --no-"""
     1483    match = re.match(r"(--no-|--|-)(.*)", option_name)
     1484    return (match.group(1), match.group(2))
     1485
     1486
     1487def _compare_option_names(a, b):
     1488    """Compares option names, sorting by name, with --no-foo after --foo."""
     1489    a_prefix, a_name = _split_option_name(a)
     1490    b_prefix, b_name = _split_option_name(b)
     1491    # Sort options first by name ignoring prefix
     1492    compare_result = cmp(a_name, b_name)
     1493    if compare_result != 0:
     1494        return compare_result
     1495    # Sort --no-foo after --foo
     1496    return cmp(a_prefix, b_prefix)
     1497
     1498
     1499def _compare_options(a, b):
     1500    """Compares options, sorting by name using _compare_option_names."""
     1501    return _compare_option_names(a.get_opt_string(), b.get_opt_string())
     1502
     1503
     1504def _add_option_group(parser, title, options):
     1505    """Sorts and groups options and then adds the group to the parser."""
     1506    group = optparse.OptionGroup(parser, title)
     1507    group.add_options(sorted(options, _compare_options))
     1508    parser.add_option_group(group)
     1509
     1510
     1511def _add_hidden_options(parser, options):
     1512    """Groups options into a HiddenOptionGroup and adds the group to the parser."""
     1513    hidden_group = HiddenOptionGroup(parser)
     1514    hidden_group.add_options(options)
     1515    parser.add_option_group(hidden_group)
     1516
     1517
    14481518def parse_args(args=None):
    14491519    """Provides a default set of command line args.
    14501520
    14511521    Returns a tuple of options, args from optparse"""
     1522
     1523    parser = optparse.OptionParser()
    14521524
    14531525    # FIXME: All of these options should be stored closer to the code which
     
    14551527    # FIXME: to WebKitPort and be shared across all scripts.
    14561528    configuration_options = [
    1457         optparse.make_option("-t", "--target", dest="configuration",
    1458                              help="(DEPRECATED)"),
    1459         # FIXME: --help should display which configuration is default.
    14601529        optparse.make_option('--debug', action='store_const', const='Debug',
    14611530                             dest="configuration",
    1462                              help='Set the configuration to Debug'),
     1531                             help="Set build configuration to Debug"),
    14631532        optparse.make_option('--release', action='store_const',
    14641533                             const='Release', dest="configuration",
    1465                              help='Set the configuration to Release'),
    1466         # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
     1534                             help="Set build configuration to Release"),
     1535        # FIXME: --help should display which configuration is default.
     1536        optparse.make_option("-c", "--configuration", action="store",
     1537                             help="Set build configuration"),
    14671538    ]
    1468 
    1469     print_options = printing.print_options()
     1539    _add_option_group(parser, "Configuration Options", configuration_options)
     1540    _add_option_group(parser, "Printing Options", printing.print_options())
    14701541
    14711542    # FIXME: These options should move onto the ChromiumPort.
    14721543    chromium_options = [
    1473         optparse.make_option("--chromium", action="store_true", default=False,
    1474             help="use the Chromium port"),
     1544        optparse.make_option("--chromium", action="store_true",
     1545            help="Use the Chromium port."),
    14751546        optparse.make_option("--startup-dialog", action="store_true",
    1476             default=False, help="create a dialog on DumpRenderTree startup"),
     1547            help="Create a dialog on test_shell startup."),
    14771548        optparse.make_option("--gp-fault-error-box", action="store_true",
    1478             default=False, help="enable Windows GP fault error box"),
     1549            help="Enable Windows GP fault error box."),
    14791550        optparse.make_option("--nocheck-sys-deps", action="store_true",
    1480             default=False,
    1481             help="Don't check the system dependencies (themes)"),
     1551            help="Disable system dependencies check (eg. fonts, themes)."),
    14821552        optparse.make_option("--use-drt", action="store_true",
    1483             default=False,
    1484             help="Use DumpRenderTree instead of test_shell"),
     1553            help="Use DumpRenderTree instead of test_shell."),
     1554        optparse.make_option("--use-apache", action="store_true",
     1555            help="Use apache instead of lighttpd (Windows only)."),
    14851556    ]
     1557    _add_option_group(parser, "Chromium Options", chromium_options)
    14861558
    14871559    # Missing Mac-specific old-run-webkit-tests options:
     
    14951567        # NRWT doesn't sample on timeout yet anyway.
    14961568        _compat_shim_option("--no-sample-on-timeout"),
    1497         # FIXME: NRWT needs to support remote links eventually.
     1569        # FIXME: NRWT needs to support remote links.
    14981570        _compat_shim_option("--use-remote-links-to-tests"),
    14991571        # FIXME: NRWT doesn't need this option as much since failures are
     
    15011573        _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"),
    15021574    ]
     1575    _add_hidden_options(parser, old_run_webkit_tests_compat)
    15031576
    15041577    results_options = [
     
    15061579        # within the SVN repository in the results.
    15071580        optparse.make_option("-p", "--pixel-tests", action="store_true",
    1508             dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
     1581            help="Enable pixel-to-pixel PNG comparisons"),
    15091582        optparse.make_option("--no-pixel-tests", action="store_false",
    15101583            dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
    1511         optparse.make_option("--fuzzy-pixel-tests", action="store_true",
    1512             default=False,
    1513             help="Also use fuzzy matching to compare pixel test outputs."),
    15141584        # old-run-webkit-tests allows a specific tolerance: --tolerance t
    15151585        # Ignore image differences less than this percentage (default: 0.1)
     1586        optparse.make_option("--fuzzy-pixel-tests", action="store_true",
     1587            help="Also use fuzzy matching to compare pixel test outputs."),
    15161588        optparse.make_option("--results-directory",
    15171589            default="layout-test-results",
    1518             help="Output results directory source dir, relative to Debug or "
    1519                  "Release"),
     1590            help="Output results directory, relative to build directory"),
    15201591        optparse.make_option("--new-baseline", action="store_true",
    1521             default=False, help="Save all generated results as new baselines "
     1592            help="Save all generated results as new baselines "
    15221593                 "into the platform directory, overwriting whatever's "
    15231594                 "already there."),
    15241595        optparse.make_option("--no-show-results", action="store_false",
    15251596            default=True, dest="show_results",
    1526             help="Don't launch a browser with results after the tests "
    1527                  "are done"),
    1528         # FIXME: We should have a helper function to do this sort of
    1529         # deprectated mapping and automatically log, etc.
    1530         optparse.make_option("--noshow-results", action="store_false",
    1531             dest="show_results",
    1532             help="Deprecated, same as --no-show-results."),
    1533         optparse.make_option("--no-launch-safari", action="store_false",
    1534             dest="show_results",
    1535             help="old-run-webkit-tests compat, same as --noshow-results."),
    1536         # old-run-webkit-tests:
    1537         # --[no-]launch-safari    Launch (or do not launch) Safari to display
    1538         #                         test results (default: launch)
     1597            help="Do not launch a browser with results after testing"),
    15391598        optparse.make_option("--full-results-html", action="store_true",
    1540             default=False,
    1541             help="Show all failures in results.html, rather than only "
    1542                  "regressions"),
    1543         optparse.make_option("--clobber-old-results", action="store_true",
    1544             default=False, help="Clobbers test results from previous runs."),
     1599            help="Include expected failures in results.html."),
     1600        optparse.make_option("--reset-results", action="store_true",
     1601            help="Reset all results with new expectations from this run."),
    15451602        optparse.make_option("--platform",
    15461603            help="Override the platform for expected results"),
     
    15531610        #                                 (default: 0)
    15541611    ]
     1612    _add_option_group(parser, "Results Options", results_options)
    15551613
    15561614    test_options = [
     
    15641622        # old-run-webkit-tests has --valgrind instead of wrapper.
    15651623        optparse.make_option("--wrapper",
    1566             help="wrapper command to insert before invocations of "
    1567                  "DumpRenderTree; option is split on whitespace before "
    1568                  "running. (Example: --wrapper='valgrind --smc-check=all')"),
     1624            help="Wrap all invocations of DumpRenderTree with WRAPPER. "
     1625                 "WRAPPER is split on whitespace before running. "
     1626                 "(Example: --wrapper='valgrind --smc-check=all')"),
    15691627        # old-run-webkit-tests:
    15701628        # -i|--ignore-tests               Comma-separated list of directories
    15711629        #                                 or tests to ignore
    15721630        optparse.make_option("--test-list", action="append",
    1573             help="read list of tests to run from file", metavar="FILE"),
     1631            help="Read list of tests to run from file.", metavar="FILE"),
    15741632        # old-run-webkit-tests uses --skipped==[default|ignore|only]
    15751633        # instead of --force:
    1576         optparse.make_option("--force", action="store_true", default=False,
     1634        optparse.make_option("--force", action="store_true",
    15771635            help="Run all tests, even those marked SKIP in the test list"),
    1578         optparse.make_option("--use-apache", action="store_true",
    1579             default=False, help="Whether to use apache instead of lighttpd."),
    1580         optparse.make_option("--time-out-ms",
    1581             help="Set the timeout for each test"),
    1582         # old-run-webkit-tests calls --randomize-order --random:
     1636        optparse.make_option("--time-out-ms", metavar="MS",
     1637            help="Set the timeout for all tests."),
    15831638        optparse.make_option("--randomize-order", action="store_true",
    1584             default=False, help=("Run tests in random order (useful "
    1585                                 "for tracking down corruption)")),
     1639            help="Run the tests in a random order."),
     1640        optparse.make_option("--singly", action="store_true", dest="run_singly",
     1641            help="Run a separate DumpRenderTree for each test"),
     1642        optparse.make_option("--nthly", dest="batch_size",
     1643            help="Restart DumpRenderTree every n tests."),
     1644        # FIXME: Display default number of child processes that will run.
     1645        optparse.make_option("--child-processes",
     1646            help="Number of DumpRenderTrees to run in parallel."),
     1647        optparse.make_option("--experimental-fully-parallel",
     1648            action="store_true", help="Run tests with per-file parallelism."),
     1649        # FIXME: Need --exit-after-n-failures N
     1650        #      Exit after the first N failures instead of running all tests
     1651        # FIXME: consider: --iterations n
     1652        #      Number of times to run the set of tests (e.g. ABCABCABC)
     1653    ]
     1654    _add_option_group(parser, "Test Options", test_options)
     1655
     1656    # These two options exist only for the Chromium valgrind bot to use.
     1657    # valgrind is too slow to run more than a few tests per build, so
     1658    # the builder runs a different small chunk each build.
     1659    test_grouping_options = [
    15861660        optparse.make_option("--run-chunk",
    15871661            help=("Run a specified chunk (n:l), the nth of len l, "
     
    15891663        optparse.make_option("--run-part", help=("Run a specified part (n:m), "
    15901664                  "the nth of m parts, of the layout tests")),
    1591         # old-run-webkit-tests calls --batch-size: --nthly n
    1592         #   Restart DumpRenderTree every n tests (default: 1000)
    1593         optparse.make_option("--batch-size",
    1594             help=("Run a the tests in batches (n), after every n tests, "
    1595                   "DumpRenderTree is relaunched.")),
    1596         # old-run-webkit-tests calls --run-singly: -1|--singly
    1597         # Isolate each test case run (implies --nthly 1 --verbose)
    1598         optparse.make_option("--run-singly", action="store_true",
    1599             default=False, help="run a separate DumpRenderTree for each test"),
    1600         optparse.make_option("--child-processes",
    1601             help="Number of DumpRenderTrees to run in parallel."),
    1602         # FIXME: Display default number of child processes that will run.
    1603         optparse.make_option("--experimental-fully-parallel",
    1604             action="store_true", default=False,
    1605             help="run all tests in parallel"),
    1606         # FIXME: Need --exit-after-n-failures N
    1607         #      Exit after the first N failures instead of running all tests
    1608         # FIXME: consider: --iterations n
    1609         #      Number of times to run the set of tests (e.g. ABCABCABC)
     1665    ]
     1666    _add_hidden_options(parser, test_grouping_options)
     1667
     1668    misc_options = [
     1669        optparse.make_option("--lint-test-files", action="store_true",
     1670        default=False, help=("Makes sure the test files parse for all "
     1671                            "configurations. Does not run any tests.")),
    16101672        optparse.make_option("--print-last-failures", action="store_true",
    16111673            default=False, help="Print the tests in the last run that "
     
    16211683            help="Don't re-try any tests that produce unexpected results."),
    16221684    ]
    1623 
    1624     misc_options = [
    1625         optparse.make_option("--lint-test-files", action="store_true",
    1626         default=False, help=("Makes sure the test files parse for all "
    1627                             "configurations. Does not run any tests.")),
    1628     ]
    1629 
    1630     # FIXME: Move these into json_results_generator.py
     1685    parser.add_options(misc_options)
     1686
     1687    # These only exist for the build bots for generating results.json output.
    16311688    results_json_options = [
    16321689        optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
    1633             help=("The name of the builder shown on the waterfall running "
    1634                   "this script e.g. WebKit.")),
     1690            help="Builder name as shown on the on the waterfall e.g. WebKit."),
    16351691        optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
    1636             help=("The name of the builder used in its path, e.g. "
    1637                   "webkit-rel.")),
     1692            help="Builder name as used in urls, e.g. webkit-rel."),
    16381693        optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
    1639             help=("The build number of the builder running this script.")),
     1694            help="Buildbot's build number for this build."),
    16401695        optparse.make_option("--test-results-server", default="",
    16411696            help=("If specified, upload results json files to this appengine "
    16421697                  "server.")),
    16431698    ]
    1644 
    1645     option_list = (configuration_options + print_options +
    1646                    chromium_options + results_options + test_options +
    1647                    misc_options + results_json_options +
    1648                    old_run_webkit_tests_compat)
    1649     option_parser = optparse.OptionParser(option_list=option_list)
    1650 
    1651     options, args = option_parser.parse_args(args)
     1699    _add_hidden_options(parser, results_json_options)
     1700
     1701    deprecated_options = [
     1702        _deprecated_option(parser, "--batch-size", new_option_name="--nthly"),
     1703        _deprecated_option(parser, "--clobber-old-results", new_option_name="--reset-results"),
     1704        _deprecated_option(parser, "--noshow-results", new_option_name="--no-show-results"),
     1705        _deprecated_option(parser, "--no-launch-safari", new_option_name="--no-show-results"),
     1706        _deprecated_option(parser, "--random", new_option_name="--randomize-order"),
     1707        _deprecated_option(parser, "--run-singly", new_option_name="--singly"),
     1708        _deprecated_option(parser, "-t", "--target", new_option_name="--configuration"),
     1709    ]
     1710    _add_hidden_options(parser, deprecated_options)
     1711
     1712    options, args = parser.parse_args(args)
    16521713    if options.sources:
    16531714        options.verbose = True
  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

    r59182 r59631  
    6666                                     'fast/html']))
    6767        self.assertTrue(passing_run(['--platform', 'test',
    68                                      '--run-singly',
     68                                     '--singly',
    6969                                     'fast/html']))
    7070        self.assertTrue(passing_run(['--platform', 'test',
     
    131131
    132132
    133 if __name__ == '__main__':
    134     unittest.main()
     133class OptionParsingTest(unittest.TestCase):
     134    def _assert_name_split(self, option_name, expected_prefix, expected_name):
     135        prefix, name = run_webkit_tests._split_option_name(option_name)
     136        self.assertEqual(prefix, expected_prefix)
     137        self.assertEqual(name, expected_name)
     138
     139    def test_option_sorting(self):
     140        self._assert_name_split("-n", "-", "n")
     141        self._assert_name_split("--name", "--", "name")
     142        self._assert_name_split("--no-name", "--no-", "name")
     143
     144        options = [
     145            "--no-abc",
     146            "-a",
     147            "--xyz",
     148            "--abc",
     149        ]
     150        expected_sort = [
     151            "-a",
     152            "--abc",
     153            "--no-abc",
     154            "--xyz",
     155        ]
     156        options.sort(run_webkit_tests._compare_option_names)
     157        self.assertEqual(options, expected_sort)
Note: See TracChangeset for help on using the changeset viewer.