Changeset 56193 in webkit


Ignore:
Timestamp:
Mar 18, 2010 2:21:55 PM (14 years ago)
Author:
dpranke@chromium.org
Message:

2010-03-17 Dirk Pranke <​dpranke@chromium.org>

Reviewed by Dimitri Glazkov.

Add the concept of an "overrides" file for expectations so that we
can store test_expectations both upstream and downstream for a port
that runs both in webkit.org and in a separate repository (like
Chromium). Also add some unit tests for the expectations module.

​https://bugs.webkit.org/show_bug.cgi?id=36249

  • Scripts/webkitpy/layout_tests/layout_package/test_expectations.py:
  • Scripts/webkitpy/layout_tests/layout_package/test_expectations_test.py: Added.
  • Scripts/webkitpy/layout_tests/port/base.py:
  • Scripts/webkitpy/layout_tests/run_webkit_tests.py:
Location:
trunk/WebKitTools
Files:
1 added
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/WebKitTools/ChangeLog

    r56183 r56193  
     12010-03-17  Dirk Pranke  <dpranke@chromium.org>
     2
     3        Reviewed by Dimitri Glazkov.
     4
     5        Add the concept of an "overrides" file for expectations so that we
     6        can store test_expectations both upstream and downstream for a port
     7        that runs both in webkit.org and in a separate repository (like
     8        Chromium). Also add some unit tests for the expectations module.
     9
     10        https://bugs.webkit.org/show_bug.cgi?id=36249
     11
     12        * Scripts/webkitpy/layout_tests/layout_package/test_expectations.py:
     13        * Scripts/webkitpy/layout_tests/layout_package/test_expectations_test.py: Added.
     14        * Scripts/webkitpy/layout_tests/port/base.py:
     15        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
     16
    1172010-03-18  Jesus Sanchez-Palencia  <jesus.palencia@openbossa.org>
    218

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py

    r55603 r56193  
    5454
    5555    def __init__(self, port, tests, expectations, test_platform_name,
    56                  is_debug_mode, is_lint_mode, tests_are_present=True):
     56                 is_debug_mode, is_lint_mode, tests_are_present=True,
     57                 overrides=None):
    5758        """Loads and parses the test expectations given in the string.
    5859        Args:
    … …  
    7172                test files from directories, and is needed by the LTTF
    7273                dashboard, where the files aren't actually locally present.
     74            overrides: test expectations that are allowed to override any
     75                entries in |expectations|. This is used by callers
     76                that need to manage two sets of expectations (e.g., upstream
     77                and downstream expectations).
    7378        """
    7479        self._expected_failures = TestExpectationsFile(port, expectations,
    7580            tests, test_platform_name, is_debug_mode, is_lint_mode,
    76             tests_are_present=tests_are_present)
     81            tests_are_present=tests_are_present, overrides=overrides)
    7782
    7883    # TODO(ojan): Allow for removing skipped tests when getting the list of
    … …  
    265270    def __init__(self, port, expectations, full_test_list, test_platform_name,
    266271        is_debug_mode, is_lint_mode, suppress_errors=False,
    267         tests_are_present=True):
     272        tests_are_present=True, overrides=None):
    268273        """
    269274        expectations: Contents of the expectations file
    … …  
    279284            filesystem. The LTTF Dashboard uses False here to avoid having to
    280285            keep a local copy of the tree.
     286        overrides: test expectations that are allowed to override any
     287            entries in |expectations|. This is used by callers
     288            that need to manage two sets of expectations (e.g., upstream
     289            and downstream expectations).
    281290        """
    282291
    … …  
    288297        self._is_lint_mode = is_lint_mode
    289298        self._tests_are_present = tests_are_present
     299        self._overrides = overrides
    290300        self._suppress_errors = suppress_errors
    291301        self._errors = []
    … …  
    315325        self._result_type_to_tests = self._dict_of_sets(self.RESULT_TYPES)
    316326
    317         self._read(self._get_iterable_expectations())
     327        self._read(self._get_iterable_expectations(self._expectations),
     328                   overrides_allowed=False)
     329
     330        # List of tests that are in the overrides file (used for checking for
     331        # duplicates inside the overrides file itself). Note that just because
     332        # a test is in this set doesn't mean it's necessarily overridding a
     333        # expectation in the regular expectations; the test might not be
     334        # mentioned in the regular expectations file at all.
     335        self._overridding_tests = set()
     336
     337        if overrides:
     338            self._read(self._get_iterable_expectations(self._overrides),
     339                       overrides_allowed=True)
     340
     341        self._handle_any_read_errors()
     342        self._process_tests_without_expectations()
     343
     344    def _handle_any_read_errors(self):
     345        if not self._suppress_errors and (
     346            len(self._errors) or len(self._non_fatal_errors)):
     347            if self._is_debug_mode:
     348                build_type = 'DEBUG'
     349            else:
     350                build_type = 'RELEASE'
     351            _log.error('')
     352            _log.error("FAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" %
     353                       (self._test_platform_name.upper(), build_type))
     354
     355            for error in self._non_fatal_errors:
     356                _log.error(error)
     357            _log.error('')
     358
     359            if len(self._errors):
     360                raise SyntaxError('\n'.join(map(str, self._errors)))
     361
     362    def _process_tests_without_expectations(self):
     363        expectations = set([PASS])
     364        options = []
     365        modifiers = []
     366        if self._full_test_list:
     367            for test in self._full_test_list:
     368                if not test in self._test_list_paths:
     369                    self._add_test(test, modifiers, expectations, options,
     370                        overrides_allowed=False)
    318371
    319372    def _dict_of_sets(self, strings_to_constants):
    … …  
    325378        return d
    326379
    327     def _get_iterable_expectations(self):
     380    def _get_iterable_expectations(self, expectations_str):
    328381        """Returns an object that can be iterated over. Allows for not caring
    329382        about whether we're iterating over a file or a new-line separated
    330383        string."""
    331         iterable = [x + "\n" for x in
    332             self._expectations.split("\n")]
     384        iterable = [x + "\n" for x in expectations_str.split("\n")]
    333385        # Strip final entry if it's empty to avoid added in an extra
    334386        # newline.
    … …  
    578630            ModifiersAndExpectations(options, expectations))
    579631
    580     def _read(self, expectations):
     632    def _read(self, expectations, overrides_allowed):
    581633        """For each test in an expectations iterable, generate the
    582634        expectations for it."""
    … …  
    629681
    630682            self._add_tests(tests, expectations, test_list_path, lineno,
    631                            modifiers, options)
    632 
    633         if not self._suppress_errors and (
    634             len(self._errors) or len(self._non_fatal_errors)):
    635             if self._is_debug_mode:
    636                 build_type = 'DEBUG'
    637             else:
    638                 build_type = 'RELEASE'
    639             _log.error('')
    640             _log.error("FAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" %
    641                        (self._test_platform_name.upper(), build_type))
    642 
    643             for error in self._non_fatal_errors:
    644                 _log.error(error)
    645             _log.error('')
    646 
    647             if len(self._errors):
    648                 raise SyntaxError('\n'.join(map(str, self._errors)))
    649 
    650         # Now add in the tests that weren't present in the expectations file
    651         expectations = set([PASS])
    652         options = []
    653         modifiers = []
    654         if self._full_test_list:
    655             for test in self._full_test_list:
    656                 if not test in self._test_list_paths:
    657                     self._add_test(test, modifiers, expectations, options)
     683                           modifiers, options, overrides_allowed)
    658684
    659685    def _get_options_list(self, listString):
    … …  
    699725
    700726    def _add_tests(self, tests, expectations, test_list_path, lineno,
    701                    modifiers, options):
     727                   modifiers, options, overrides_allowed):
    702728        for test in tests:
    703             if self._already_seen_test(test, test_list_path, lineno):
     729            if self._already_seen_test(test, test_list_path, lineno,
     730                                       overrides_allowed):
    704731                continue
    705732
    706733            self._clear_expectations_for_test(test, test_list_path)
    707             self._add_test(test, modifiers, expectations, options)
    708 
    709     def _add_test(self, test, modifiers, expectations, options):
     734            self._add_test(test, modifiers, expectations, options,
     735                           overrides_allowed)
     736
     737    def _add_test(self, test, modifiers, expectations, options,
     738                  overrides_allowed):
    710739        """Sets the expected state for a given test.
    711740
    … …  
    718747          modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.)
    719748          expectations: sequence of expectations (PASS, IMAGE, etc.)
    720           options: sequence of keywords and bug identifiers."""
     749          options: sequence of keywords and bug identifiers.
     750          overrides_allowed: whether we're parsing the regular expectations
     751              or the overridding expectations"""
    721752        self._test_to_expectations[test] = expectations
    722753        for expectation in expectations:
    … …  
    745776        else:
    746777            self._result_type_to_tests[FAIL].add(test)
     778
     779        if overrides_allowed:
     780            self._overridding_tests.add(test)
    747781
    748782    def _clear_expectations_for_test(self, test, test_list_path):
    … …  
    770804                set_of_tests.remove(test)
    771805
    772     def _already_seen_test(self, test, test_list_path, lineno):
     806    def _already_seen_test(self, test, test_list_path, lineno, allow_overrides):
    773807        """Returns true if we've already seen a more precise path for this test
    774808        than the test_list_path.
    … …  
    779813        prev_base_path = self._test_list_paths[test]
    780814        if (prev_base_path == os.path.normpath(test_list_path)):
    781             self._add_error(lineno, 'Duplicate expectations.', test)
    782             return True
     815            if (not allow_overrides or test in self._overridding_tests):
     816                if allow_overrides:
     817                    expectation_source = "override"
     818                else:
     819                    expectation_source = "expectation"
     820                self._add_error(lineno, 'Duplicate %s.' % expectation_source,
     821                                   test)
     822                return True
     823            else:
     824                # We have seen this path, but that's okay because its
     825                # in the overrides and the earlier path was in the
     826                # expectations.
     827                return False
    783828
    784829        # Check if we've already seen a more precise path.

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py

    r56123 r56193  
    441441        test_expectations file. See test_expectations.py for more details."""
    442442        raise NotImplementedError('Port.test_expectations')
     443
     444    def test_expectations_overrides(self):
     445        """Returns an optional set of overrides for the test_expectations.
     446
     447        This is used by ports that have code in two repositories, and where
     448        it is possible that you might need "downstream" expectations that
     449        temporarily override the "upstream" expectations until the port can
     450        sync up the two repos."""
     451        return None
    443452
    444453    def test_base_platform_names(self):

  • trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

    r55938 r56193  
    228228        try:
    229229            expectations_str = self._port.test_expectations()
     230            overrides_str = self._port.test_expectations_overrides()
    230231            self._expectations = test_expectations.TestExpectations(
    231232                self._port, test_files, expectations_str, test_platform_name,
    232                 is_debug_mode, self._options.lint_test_files)
     233                is_debug_mode, self._options.lint_test_files,
     234                tests_are_present=True, overrides=overrides_str)
    233235            return self._expectations
    234236        except Exception, err:
    … …  
    14611463            test_runner.parse_expectations(platform, is_debug_mode=True)
    14621464            test_runner.parse_expectations(platform, is_debug_mode=False)
     1465        meter.update("")
    14631466        print ("If there are no fail messages, errors or exceptions, then the "
    14641467            "lint succeeded.")
Note: See TracChangeset for help on using the changeset viewer.