Changeset 59631 in webkit
- Timestamp:
- May 17, 2010 5:15:16 PM (14 years ago)
- Location:
- trunk/WebKitTools
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/WebKitTools/ChangeLog
r59623 r59631 1 2010-05-17 Eric Seidel <eric@webkit.org> 2 3 Reviewed by Adam Barth. 4 5 Attempt to make new-run-webkit-tests --help more sane 6 https://bugs.webkit.org/show_bug.cgi?id=37836 7 8 * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py: 9 - Add a FIXME about options.singly and options.batch_size being different. 10 * Scripts/webkitpy/layout_tests/run_webkit_tests.py: 11 - Add support for hidden options. 12 - Add option groupings to attempt to simplify --help. 13 - Fix a bunch of option helps to start with a capitalized verb. 14 - Hide a bunch of options which make no sense to users. 15 - Sort options in --help. 16 * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py: 17 - Add tests for option sorting. 18 1 19 2010-05-17 Robert Hogan <robert@webkit.org> 2 20 -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
r58473 r59631 325 325 batch_count += 1 326 326 self._num_tests += 1 327 # FIXME: options.run_singly and options.batch_size=1 should use the same code path! 327 328 if self._options.run_singly: 328 329 result = self._run_test_singly(test_info) -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
r59108 r59631 1326 1326 return 0 1327 1327 1328 if options. clobber_old_results:1328 if options.reset_results: 1329 1329 # Just clobber the actual test results directories since the other 1330 1330 # files in the results directory are explicitly used for cross-run … … 1436 1436 1437 1437 1438 class HiddenOptionGroup(optparse.OptionGroup): 1439 def __init__(self, parser): 1440 optparse.OptionGroup.__init__(self, parser, title=None) 1441 1442 def format_help(self, formatter): 1443 return "" 1444 1445 1438 1446 def _compat_shim_callback(option, opt_str, value, parser): 1447 """Logs that an option is being ignored.""" 1439 1448 print "Ignoring unsupported option: %s" % opt_str 1440 1449 1441 1450 1442 1451 def _compat_shim_option(option_name, **kwargs): 1452 """returns an optparse Option which simply logs that it's being ignored.""" 1443 1453 return optparse.make_option(option_name, action="callback", 1444 1454 callback=_compat_shim_callback, … … 1446 1456 1447 1457 1458 def _deprecated_shim_callback(option, opt_str, value, parser, new_option): 1459 """Logs that the option is deprecated and then calls new_option.process""" 1460 print "%s is DEPRECATED please use %s" % (opt_str, new_option) 1461 # We could time.sleep here or some other discouraging measure. 1462 new_option.process(opt_str, value, parser.values, parser) 1463 1464 1465 def _deprecated_option(parser, option_name, long_name=None, new_option_name=None): 1466 """Returns an option which maps from and old new to a new name and prints 1467 a deprecation warning when called.""" 1468 new_option = parser.get_option(new_option_name) 1469 option_names = [option_name] 1470 if long_name: 1471 option_names.append(long_name) 1472 return optparse.make_option( 1473 action="callback", 1474 callback=_deprecated_shim_callback, 1475 callback_args=(new_option,), 1476 nargs=new_option.nargs, 1477 help="DEPRECATED, use %s" % new_option, 1478 *option_names) 1479 1480 1481 def _split_option_name(option_name): 1482 """Returns the name of the option, ignoring any leading -, --, or --no-""" 1483 match = re.match(r"(--no-|--|-)(.*)", option_name) 1484 return (match.group(1), match.group(2)) 1485 1486 1487 def _compare_option_names(a, b): 1488 """Compares option names, sorting by name, with --no-foo after --foo.""" 1489 a_prefix, a_name = _split_option_name(a) 1490 b_prefix, b_name = _split_option_name(b) 1491 # Sort options first by name ignoring prefix 1492 compare_result = cmp(a_name, b_name) 1493 if compare_result != 0: 1494 return compare_result 1495 # Sort --no-foo after --foo 1496 return cmp(a_prefix, b_prefix) 1497 1498 1499 def _compare_options(a, b): 1500 """Compares options, sorting by name using _compare_option_names.""" 1501 return _compare_option_names(a.get_opt_string(), b.get_opt_string()) 1502 1503 1504 def _add_option_group(parser, title, options): 1505 """Sorts and groups options and then adds the group to the parser.""" 1506 group = optparse.OptionGroup(parser, title) 1507 group.add_options(sorted(options, _compare_options)) 1508 parser.add_option_group(group) 1509 1510 1511 def _add_hidden_options(parser, options): 1512 """Groups options into a HiddenOptionGroup and adds the group to the parser.""" 1513 hidden_group = HiddenOptionGroup(parser) 1514 hidden_group.add_options(options) 1515 parser.add_option_group(hidden_group) 1516 1517 1448 1518 def parse_args(args=None): 1449 1519 """Provides a default set of command line args. 1450 1520 1451 1521 Returns a tuple of options, args from optparse""" 1522 1523 parser = optparse.OptionParser() 1452 1524 1453 1525 # FIXME: All of these options should be stored closer to the code which … … 1455 1527 # FIXME: to WebKitPort and be shared across all scripts. 1456 1528 configuration_options = [ 1457 optparse.make_option("-t", "--target", dest="configuration",1458 help="(DEPRECATED)"),1459 # FIXME: --help should display which configuration is default.1460 1529 optparse.make_option('--debug', action='store_const', const='Debug', 1461 1530 dest="configuration", 1462 help= 'Set the configuration to Debug'),1531 help="Set build configuration to Debug"), 1463 1532 optparse.make_option('--release', action='store_const', 1464 1533 const='Release', dest="configuration", 1465 help='Set the configuration to Release'), 1466 # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION. 1534 help="Set build configuration to Release"), 1535 # FIXME: --help should display which configuration is default. 1536 optparse.make_option("-c", "--configuration", action="store", 1537 help="Set build configuration"), 1467 1538 ] 1468 1469 print_options = printing.print_options()1539 _add_option_group(parser, "Configuration Options", configuration_options) 1540 _add_option_group(parser, "Printing Options", printing.print_options()) 1470 1541 1471 1542 # FIXME: These options should move onto the ChromiumPort. 1472 1543 chromium_options = [ 1473 optparse.make_option("--chromium", action="store_true", default=False,1474 help=" use the Chromium port"),1544 optparse.make_option("--chromium", action="store_true", 1545 help="Use the Chromium port."), 1475 1546 optparse.make_option("--startup-dialog", action="store_true", 1476 default=False, help="create a dialog on DumpRenderTree startup"),1547 help="Create a dialog on test_shell startup."), 1477 1548 optparse.make_option("--gp-fault-error-box", action="store_true", 1478 default=False, help="enable Windows GP fault error box"),1549 help="Enable Windows GP fault error box."), 1479 1550 optparse.make_option("--nocheck-sys-deps", action="store_true", 1480 default=False, 1481 help="Don't check the system dependencies (themes)"), 1551 help="Disable system dependencies check (eg. fonts, themes)."), 1482 1552 optparse.make_option("--use-drt", action="store_true", 1483 default=False, 1484 help="Use DumpRenderTree instead of test_shell"), 1553 help="Use DumpRenderTree instead of test_shell."), 1554 optparse.make_option("--use-apache", action="store_true", 1555 help="Use apache instead of lighttpd (Windows only)."), 1485 1556 ] 1557 _add_option_group(parser, "Chromium Options", chromium_options) 1486 1558 1487 1559 # Missing Mac-specific old-run-webkit-tests options: … … 1495 1567 # NRWT doesn't sample on timeout yet anyway. 1496 1568 _compat_shim_option("--no-sample-on-timeout"), 1497 # FIXME: NRWT needs to support remote links eventually.1569 # FIXME: NRWT needs to support remote links. 1498 1570 _compat_shim_option("--use-remote-links-to-tests"), 1499 1571 # FIXME: NRWT doesn't need this option as much since failures are … … 1501 1573 _compat_shim_option("--exit-after-n-failures", nargs=1, type="int"), 1502 1574 ] 1575 _add_hidden_options(parser, old_run_webkit_tests_compat) 1503 1576 1504 1577 results_options = [ … … 1506 1579 # within the SVN repository in the results. 1507 1580 optparse.make_option("-p", "--pixel-tests", action="store_true", 1508 dest="pixel_tests",help="Enable pixel-to-pixel PNG comparisons"),1581 help="Enable pixel-to-pixel PNG comparisons"), 1509 1582 optparse.make_option("--no-pixel-tests", action="store_false", 1510 1583 dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"), 1511 optparse.make_option("--fuzzy-pixel-tests", action="store_true",1512 default=False,1513 help="Also use fuzzy matching to compare pixel test outputs."),1514 1584 # old-run-webkit-tests allows a specific tolerance: --tolerance t 1515 1585 # Ignore image differences less than this percentage (default: 0.1) 1586 optparse.make_option("--fuzzy-pixel-tests", action="store_true", 1587 help="Also use fuzzy matching to compare pixel test outputs."), 1516 1588 optparse.make_option("--results-directory", 1517 1589 default="layout-test-results", 1518 help="Output results directory source dir, relative to Debug or " 1519 "Release"), 1590 help="Output results directory, relative to build directory"), 1520 1591 optparse.make_option("--new-baseline", action="store_true", 1521 default=False,help="Save all generated results as new baselines "1592 help="Save all generated results as new baselines " 1522 1593 "into the platform directory, overwriting whatever's " 1523 1594 "already there."), 1524 1595 optparse.make_option("--no-show-results", action="store_false", 1525 1596 default=True, dest="show_results", 1526 help="Don't launch a browser with results after the tests " 1527 "are done"), 1528 # FIXME: We should have a helper function to do this sort of 1529 # deprectated mapping and automatically log, etc. 1530 optparse.make_option("--noshow-results", action="store_false", 1531 dest="show_results", 1532 help="Deprecated, same as --no-show-results."), 1533 optparse.make_option("--no-launch-safari", action="store_false", 1534 dest="show_results", 1535 help="old-run-webkit-tests compat, same as --noshow-results."), 1536 # old-run-webkit-tests: 1537 # --[no-]launch-safari Launch (or do not launch) Safari to display 1538 # test results (default: launch) 1597 help="Do not launch a browser with results after testing"), 1539 1598 optparse.make_option("--full-results-html", action="store_true", 1540 default=False, 1541 help="Show all failures in results.html, rather than only " 1542 "regressions"), 1543 optparse.make_option("--clobber-old-results", action="store_true", 1544 default=False, help="Clobbers test results from previous runs."), 1599 help="Include expected failures in results.html."), 1600 optparse.make_option("--reset-results", action="store_true", 1601 help="Reset all results with new expectations from this run."), 1545 1602 optparse.make_option("--platform", 1546 1603 help="Override the platform for expected results"), … … 1553 1610 # (default: 0) 1554 1611 ] 1612 _add_option_group(parser, "Results Options", results_options) 1555 1613 1556 1614 test_options = [ … … 1564 1622 # old-run-webkit-tests has --valgrind instead of wrapper. 1565 1623 optparse.make_option("--wrapper", 1566 help=" wrapper command to insert before invocations of"1567 " DumpRenderTree; option is split on whitespace before"1568 " running.(Example: --wrapper='valgrind --smc-check=all')"),1624 help="Wrap all invocations of DumpRenderTree with WRAPPER. " 1625 "WRAPPER is split on whitespace before running. " 1626 "(Example: --wrapper='valgrind --smc-check=all')"), 1569 1627 # old-run-webkit-tests: 1570 1628 # -i|--ignore-tests Comma-separated list of directories 1571 1629 # or tests to ignore 1572 1630 optparse.make_option("--test-list", action="append", 1573 help=" read list of tests to run from file", metavar="FILE"),1631 help="Read list of tests to run from file.", metavar="FILE"), 1574 1632 # old-run-webkit-tests uses --skipped==[default|ignore|only] 1575 1633 # instead of --force: 1576 optparse.make_option("--force", action="store_true", default=False,1634 optparse.make_option("--force", action="store_true", 1577 1635 help="Run all tests, even those marked SKIP in the test list"), 1578 optparse.make_option("--use-apache", action="store_true", 1579 default=False, help="Whether to use apache instead of lighttpd."), 1580 optparse.make_option("--time-out-ms", 1581 help="Set the timeout for each test"), 1582 # old-run-webkit-tests calls --randomize-order --random: 1636 optparse.make_option("--time-out-ms", metavar="MS", 1637 help="Set the timeout for all tests."), 1583 1638 optparse.make_option("--randomize-order", action="store_true", 1584 default=False, help=("Run tests in random order (useful " 1585 "for tracking down corruption)")), 1639 help="Run the tests in a random order."), 1640 optparse.make_option("--singly", action="store_true", dest="run_singly", 1641 help="Run a separate DumpRenderTree for each test"), 1642 optparse.make_option("--nthly", dest="batch_size", 1643 help="Restart DumpRenderTree every n tests."), 1644 # FIXME: Display default number of child processes that will run. 1645 optparse.make_option("--child-processes", 1646 help="Number of DumpRenderTrees to run in parallel."), 1647 optparse.make_option("--experimental-fully-parallel", 1648 action="store_true", help="Run tests with per-file parallelism."), 1649 # FIXME: Need --exit-after-n-failures N 1650 # Exit after the first N failures instead of running all tests 1651 # FIXME: consider: --iterations n 1652 # Number of times to run the set of tests (e.g. ABCABCABC) 1653 ] 1654 _add_option_group(parser, "Test Options", test_options) 1655 1656 # These two options exist only for the Chromium valgrind bot to use. 1657 # valgrind is too slow to run more than a few tests per build, so 1658 # the builder runs a different small chunk each build. 1659 test_grouping_options = [ 1586 1660 optparse.make_option("--run-chunk", 1587 1661 help=("Run a specified chunk (n:l), the nth of len l, " … … 1589 1663 optparse.make_option("--run-part", help=("Run a specified part (n:m), " 1590 1664 "the nth of m parts, of the layout tests")), 1591 # old-run-webkit-tests calls --batch-size: --nthly n 1592 # Restart DumpRenderTree every n tests (default: 1000) 1593 optparse.make_option("--batch-size", 1594 help=("Run a the tests in batches (n), after every n tests, " 1595 "DumpRenderTree is relaunched.")), 1596 # old-run-webkit-tests calls --run-singly: -1|--singly 1597 # Isolate each test case run (implies --nthly 1 --verbose) 1598 optparse.make_option("--run-singly", action="store_true", 1599 default=False, help="run a separate DumpRenderTree for each test"), 1600 optparse.make_option("--child-processes", 1601 help="Number of DumpRenderTrees to run in parallel."), 1602 # FIXME: Display default number of child processes that will run. 1603 optparse.make_option("--experimental-fully-parallel", 1604 action="store_true", default=False, 1605 help="run all tests in parallel"), 1606 # FIXME: Need --exit-after-n-failures N 1607 # Exit after the first N failures instead of running all tests 1608 # FIXME: consider: --iterations n 1609 # Number of times to run the set of tests (e.g. ABCABCABC) 1665 ] 1666 _add_hidden_options(parser, test_grouping_options) 1667 1668 misc_options = [ 1669 optparse.make_option("--lint-test-files", action="store_true", 1670 default=False, help=("Makes sure the test files parse for all " 1671 "configurations. Does not run any tests.")), 1610 1672 optparse.make_option("--print-last-failures", action="store_true", 1611 1673 default=False, help="Print the tests in the last run that " … … 1621 1683 help="Don't re-try any tests that produce unexpected results."), 1622 1684 ] 1623 1624 misc_options = [ 1625 optparse.make_option("--lint-test-files", action="store_true", 1626 default=False, help=("Makes sure the test files parse for all " 1627 "configurations. Does not run any tests.")), 1628 ] 1629 1630 # FIXME: Move these into json_results_generator.py 1685 parser.add_options(misc_options) 1686 1687 # These only exist for the build bots for generating results.json output. 1631 1688 results_json_options = [ 1632 1689 optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME", 1633 help=("The name of the builder shown on the waterfall running " 1634 "this script e.g. WebKit.")), 1690 help="Builder name as shown on the on the waterfall e.g. WebKit."), 1635 1691 optparse.make_option("--build-name", default="DUMMY_BUILD_NAME", 1636 help=("The name of the builder used in its path, e.g. " 1637 "webkit-rel.")), 1692 help="Builder name as used in urls, e.g. webkit-rel."), 1638 1693 optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER", 1639 help= ("The build number of the builder running this script.")),1694 help="Buildbot's build number for this build."), 1640 1695 optparse.make_option("--test-results-server", default="", 1641 1696 help=("If specified, upload results json files to this appengine " 1642 1697 "server.")), 1643 1698 ] 1644 1645 option_list = (configuration_options + print_options + 1646 chromium_options + results_options + test_options + 1647 misc_options + results_json_options + 1648 old_run_webkit_tests_compat) 1649 option_parser = optparse.OptionParser(option_list=option_list) 1650 1651 options, args = option_parser.parse_args(args) 1699 _add_hidden_options(parser, results_json_options) 1700 1701 deprecated_options = [ 1702 _deprecated_option(parser, "--batch-size", new_option_name="--nthly"), 1703 _deprecated_option(parser, "--clobber-old-results", new_option_name="--reset-results"), 1704 _deprecated_option(parser, "--noshow-results", new_option_name="--no-show-results"), 1705 _deprecated_option(parser, "--no-launch-safari", new_option_name="--no-show-results"), 1706 _deprecated_option(parser, "--random", new_option_name="--randomize-order"), 1707 _deprecated_option(parser, "--run-singly", new_option_name="--singly"), 1708 _deprecated_option(parser, "-t", "--target", new_option_name="--configuration"), 1709 ] 1710 _add_hidden_options(parser, deprecated_options) 1711 1712 options, args = parser.parse_args(args) 1652 1713 if options.sources: 1653 1714 options.verbose = True -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
r59182 r59631 66 66 'fast/html'])) 67 67 self.assertTrue(passing_run(['--platform', 'test', 68 '-- run-singly',68 '--singly', 69 69 'fast/html'])) 70 70 self.assertTrue(passing_run(['--platform', 'test', … … 131 131 132 132 133 if __name__ == '__main__': 134 unittest.main() 133 class OptionParsingTest(unittest.TestCase): 134 def _assert_name_split(self, option_name, expected_prefix, expected_name): 135 prefix, name = run_webkit_tests._split_option_name(option_name) 136 self.assertEqual(prefix, expected_prefix) 137 self.assertEqual(name, expected_name) 138 139 def test_option_sorting(self): 140 self._assert_name_split("-n", "-", "n") 141 self._assert_name_split("--name", "--", "name") 142 self._assert_name_split("--no-name", "--no-", "name") 143 144 options = [ 145 "--no-abc", 146 "-a", 147 "--xyz", 148 "--abc", 149 ] 150 expected_sort = [ 151 "-a", 152 "--abc", 153 "--no-abc", 154 "--xyz", 155 ] 156 options.sort(run_webkit_tests._compare_option_names) 157 self.assertEqual(options, expected_sort)
Note: See TracChangeset
for help on using the changeset viewer.