Changeset 129051 in webkit

Timestamp:

Sep 19, 2012 3:29:45 PM (12 years ago)

Author:

dpranke@chromium.org

Message:

implement first part of support for the new TestExpectations syntax
​https://bugs.webkit.org/show_bug.cgi?id=96569

Reviewed by Ryosuke Niwa.

This patch implements support for parsing a line of the new
format for the TestExpectations file and converting it back into
the old format for compatibility. This routine is not yet used
by anything.

The new format is documented at:

​http://trac.webkit.org/wiki/TestExpectations

but, in short:

[bugs] [ "modifiers?" ] test_name [ "expectations?" ]

• Comments are indicated with "#" instead of ""
• If no expectations are specified we default to Skip for compatibility with the Skipped files (these two changes make Skipped files a subset of TestExpectations files)

• All of the tokens are now CamelCase instead of ALLCAPS.
• FAIL -> Failure
• IMAGE -> ImageOnlyFailure
• WONTFIX -> WontFix
• modifiers refer to just the platforms and configurations (release/debug) that the line applies to.
• WontFix, Rebaseline, Slow, and Skip move to the right-hand side as expectations
• expectations will typically be written out in lexicographic order
• We use webkit.org/b/12345, crbug.com/12345, and Bug(dpranke) instead of BUGWK12345, BUGCR12345, and BUGDPRANKE.

• Scripts/webkitpy/layout_tests/models/test_expectations.py:

(TestExpectationParser):
(TestExpectationParser._tokenize_line_using_new_format):

• Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:

(NewExpectationSyntaxTests):
(NewExpectationSyntaxTests.assert_exp):
(NewExpectationSyntaxTests.test_bare_name):
(NewExpectationSyntaxTests.test_bare_name_and_bugs):
(NewExpectationSyntaxTests.test_comments):
(NewExpectationSyntaxTests.test_config_modifiers):
(NewExpectationSyntaxTests.test_unknown_config):
(NewExpectationSyntaxTests.test_unknown_expectation):
(NewExpectationSyntaxTests.test_skip):
(NewExpectationSyntaxTests.test_slow):
(NewExpectationSyntaxTests.test_wontfix):
(NewExpectationSyntaxTests.test_blank_line):
(NewExpectationSyntaxTests.test_warnings):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/models/test_expectations.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-09-19  Dirk Pranke  <dpranke@chromium.org>
+
+        implement first part of support for the new TestExpectations syntax
+        https://bugs.webkit.org/show_bug.cgi?id=96569
+
+        Reviewed by Ryosuke Niwa.
+
+        This patch implements support for parsing a line of the new
+        format for the TestExpectations file and converting it back into
+        the old format for compatibility. This routine is not yet used
+        by anything.
+
+        The new format is documented at:
+          http://trac.webkit.org/wiki/TestExpectations
+
+        but, in short:
+
+          [bugs] [ "[" modifiers "]" ] test_name [ "[" expectations "]" ]
+
+        - Comments are indicated with "#" instead of "//"
+        - If no expectations are specified we default to Skip for
+          compatibility with the Skipped files (these two changes make
+          Skipped files a subset of TestExpectations files)
+
+        - All of the tokens are now CamelCase instead of ALLCAPS.
+        - FAIL -> Failure
+        - IMAGE -> ImageOnlyFailure
+        - WONTFIX -> WontFix
+        - modifiers refer to just the platforms and configurations
+          (release/debug) that the line applies to.
+        - WontFix, Rebaseline, Slow, and Skip move to the right-hand side as
+          expectations
+        - expectations will typically be written out in lexicographic order
+        - We use webkit.org/b/12345, crbug.com/12345, and Bug(dpranke)
+          instead of BUGWK12345, BUGCR12345, and BUGDPRANKE.
+
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+        (TestExpectationParser):
+        (TestExpectationParser._tokenize_line_using_new_format):
+        * Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py:
+        (NewExpectationSyntaxTests):
+        (NewExpectationSyntaxTests.assert_exp):
+        (NewExpectationSyntaxTests.test_bare_name):
+        (NewExpectationSyntaxTests.test_bare_name_and_bugs):
+        (NewExpectationSyntaxTests.test_comments):
+        (NewExpectationSyntaxTests.test_config_modifiers):
+        (NewExpectationSyntaxTests.test_unknown_config):
+        (NewExpectationSyntaxTests.test_unknown_expectation):
+        (NewExpectationSyntaxTests.test_skip):
+        (NewExpectationSyntaxTests.test_slow):
+        (NewExpectationSyntaxTests.test_wontfix):
+        (NewExpectationSyntaxTests.test_blank_line):
+        (NewExpectationSyntaxTests.test_warnings):
+
 2012-09-19  Dirk Pranke  <dpranke@chromium.org>
 

trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

@@ -255 +255 @@
         return expectation_line
 
+    # FIXME: Update the original modifiers and remove this once the old syntax is gone.
+    _configuration_tokens_list = [
+        'Mac', 'SnowLeopard', 'Lion', 'MountainLion',
+        'Win', 'XP', 'Vista', 'Win7',
+        'Linux',
+        'Android',
+        'Release',
+        'Debug',
+    ]
+
+    _configuration_tokens = dict((token, token.upper()) for token in _configuration_tokens_list)
+
+    # Note: we can't distinguish audio failures or image+text failures from text-only failures.
+    # FIXME: Update the original modifiers list and remove this once the old syntax is gone.
+    _expectation_tokens = {
+        'WontFix': 'WONTFIX',
+        'Pass': 'PASS',
+        'Failure': 'FAIL',
+        'ImageOnlyFailure': 'IMAGE',
+        'Crash': 'CRASH',
+        'Timeout': 'TIMEOUT',
+        'Slow': 'SLOW',
+    }
+
     @classmethod
     def _tokenize_line_using_new_format(cls, filename, expectation_string, line_number):
-        # FIXME: implement :).
-        raise NotImplementedError
+        """Tokenizes a line from TestExpectations and returns an unparsed TestExpectationLine instance using the old format.
+
+        The new format for a test expectation line is:
+
+        [[bugs] [ "[" <configuration modifiers> "]" <name> [ "[" <expectations> "]" ["#" <comment>]
+
+        Any errant whitespace is not preserved.
+
+        """
+        expectation_line = TestExpectationLine()
+        expectation_line.filename = filename
+        expectation_line.line_number = line_number
+
+        comment_index = expectation_string.find("#")
+        if comment_index == -1:
+            comment_index = len(expectation_string)
+        else:
+            expectation_line.comment = expectation_string[comment_index + 1:]
+
+        remaining_string = re.sub(r"\s+", " ", expectation_string[:comment_index].strip())
+        if len(remaining_string) == 0:
+            return expectation_line
+
+        # special-case parsing this so that we fail immediately instead of treating this as a test name
+        if remaining_string.startswith('//'):
+            expectation_line.warnings = ['use "#" instead of "//" for comments']
+            return expectation_line
+
+        bugs = []
+        modifiers = []
+        name = None
+        expectations = []
+        warnings = []
+
+        WEBKIT_BUG_PREFIX = 'webkit.org/b/'
+        CHROMIUM_BUG_PREFIX = 'crbug.com/'
+        V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id='
+
+        tokens = remaining_string.split()
+        state = 'start'
+        for token in tokens:
+            if (token.startswith(WEBKIT_BUG_PREFIX) or
+                token.startswith(CHROMIUM_BUG_PREFIX) or
+                token.startswith(V8_BUG_PREFIX) or
+                token.startswith('Bug(')):
+                if state != 'start':
+                    warnings.append('"%s" is not at the start of the line.' % token)
+                    break
+                if token.startswith(WEBKIT_BUG_PREFIX):
+                    bugs.append(token.replace(WEBKIT_BUG_PREFIX, 'BUGWK'))
+                elif token.startswith(CHROMIUM_BUG_PREFIX):
+                    bugs.append(token.replace(CHROMIUM_BUG_PREFIX, 'BUGCR'))
+                elif token.startswith(V8_BUG_PREFIX):
+                    bugs.append(token.replace(V8_BUG_PREFIX, 'BUGV8_'))
+                else:
+                    match = re.match('Bug\((\w+)\)$', token)
+                    if not match:
+                        warnings.append('unrecognized bug identifier "%s"' % token)
+                        break
+                    else:
+                        bugs.append('BUG' + match.group(1).upper())
+            elif token.startswith('BUG'):
+                warnings.append('unrecognized old-style bug identifier "%s"' % token)
+                break
+            elif token == '[':
+                if state == 'start':
+                    state = 'configuration'
+                elif state == 'name_found':
+                    state = 'expectations'
+                else:
+                    warnings.append('unexpected "["')
+                    break
+            elif token == ']':
+                if state == 'configuration':
+                    state = 'name'
+                elif state == 'expectations':
+                    state = 'done'
+                else:
+                    warnings.append('unexpected "]"')
+                    break
+            elif token in ('//', ':', '='):
+                warnings.append('"%s" is not legal in the new TestExpectations syntax.' % token)
+                break
+            elif state == 'configuration':
+                modifiers.append(cls._configuration_tokens.get(token, token))
+            elif state == 'expectations':
+                if token in ('Rebaseline', 'Skip', 'Slow', 'WontFix'):
+                    modifiers.append(token.upper())
+                else:
+                    expectations.append(cls._expectation_tokens.get(token, token))
+            elif state == 'name_found':
+                warnings.append('expecting "[", "#", or end of line instead of "%s"' % token)
+                break
+            else:
+                name = token
+                state = 'name_found'
+
+        if not warnings:
+            if not name:
+                warnings.append('Did not find a test name.')
+            elif state not in ('name_found', 'done'):
+                warnings.append('Missing a "]"')
+
+        if not expectations:
+            if 'SKIP' not in modifiers and 'REBASELINE' not in modifiers and 'SLOW' not in modifiers:
+                modifiers.append('SKIP')
+            expectations = ['PASS']
+
+        # FIXME: expectation line should just store bugs and modifiers separately.
+        expectation_line.modifiers = bugs + modifiers
+        expectation_line.expectations = expectations
+        expectation_line.name = name
+        expectation_line.warnings = warnings
+        return expectation_line
 
     @classmethod

trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py

@@ -341 +341 @@
 
 
+class NewExpectationSyntaxTests(unittest.TestCase):
+    def assert_exp(self, line, bugs=None, modifiers=None, expectations=None, warnings=None, comment=None, name='foo.html'):
+        bugs = bugs or []
+        modifiers = modifiers or []
+        expectations = expectations or []
+        warnings = warnings or []
+        filename = 'TestExpectations'
+        line_number = 1
+        expectation_line = TestExpectationParser._tokenize_line_using_new_format(filename, line, line_number)
+        self.assertEquals(expectation_line.warnings, warnings)
+        self.assertEquals(expectation_line.name, name)
+        self.assertEquals(expectation_line.filename, filename)
+        self.assertEquals(expectation_line.line_number, line_number)
+        if not warnings:
+            self.assertEquals(expectation_line.modifiers, modifiers)
+            self.assertEquals(expectation_line.expectations, expectations)
+
+    def test_bare_name(self):
+        self.assert_exp('foo.html', modifiers=['SKIP'], expectations=['PASS'])
+
+    def test_bare_name_and_bugs(self):
+        self.assert_exp('webkit.org/b/12345 foo.html', modifiers=['BUGWK12345', 'SKIP'], expectations=['PASS'])
+        self.assert_exp('crbug.com/12345 foo.html', modifiers=['BUGCR12345', 'SKIP'], expectations=['PASS'])
+        self.assert_exp('Bug(dpranke) foo.html', modifiers=['BUGDPRANKE', 'SKIP'], expectations=['PASS'])
+        self.assert_exp('crbug.com/12345 crbug.com/34567 foo.html', modifiers=['BUGCR12345', 'BUGCR34567', 'SKIP'], expectations=['PASS'])
+
+    def test_comments(self):
+        self.assert_exp("# comment", name=None, comment="# comment")
+        self.assert_exp("foo.html # comment", comment="# comment", expectations=['PASS'], modifiers=['SKIP'])
+
+    def test_config_modifiers(self):
+        self.assert_exp('[ Mac ] foo.html', modifiers=['MAC', 'SKIP'], expectations=['PASS'])
+        self.assert_exp('[ Mac Vista ] foo.html', modifiers=['MAC', 'VISTA', 'SKIP'], expectations=['PASS'])
+        self.assert_exp('[ Mac ] foo.html [ Failure ] ', modifiers=['MAC'], expectations=['FAIL'])
+
+    def test_unknown_config(self):
+        self.assert_exp('[ Foo ] foo.html ', modifiers=['Foo', 'SKIP'], expectations=['PASS'])
+
+    def test_unknown_expectation(self):
+        self.assert_exp('foo.html [ Audio ]', expectations=['Audio'])
+
+    def test_skip(self):
+        self.assert_exp('foo.html [ Skip ]', modifiers=['SKIP'], expectations=['PASS'])
+
+    def test_slow(self):
+        self.assert_exp('foo.html [ Slow ]', modifiers=['SLOW'], expectations=['PASS'])
+
+    def test_wontfix(self):
+        self.assert_exp('foo.html [ WontFix ]', modifiers=['WONTFIX', 'SKIP'], expectations=['PASS'])
+
+    def test_blank_line(self):
+        self.assert_exp('', name=None)
+
+    def test_warnings(self):
+        self.assert_exp('[ Mac ]', warnings=['Did not find a test name.'], name=None)
+
+        self.assert_exp('[ [', warnings=['unexpected "["'], name=None)
+        self.assert_exp('crbug.com/12345 ]', warnings=['unexpected "]"'], name=None)
+
+        self.assert_exp('foo.html crbug.com/12345 ]', warnings=['"crbug.com/12345" is not at the start of the line.'])
+
+
 class SemanticTests(Base):
     def test_bug_format(self):