Changeset 106038 in webkit

Timestamp:

Jan 26, 2012 1:46:36 PM (12 years ago)

Author:

ojan@chromium.org

Message:

Parsing test_expecations.txt + Skipped lists takes too long
​https://bugs.webkit.org/show_bug.cgi?id=77059

Reviewed by Dirk Pranke.

This saves ~100ms on the Apple Mac port.
-memoize a bunch of path methods.
-Avoid doing multiple disk accesses per line.
-Parse the skipped list directly instead of turning it into a test_expecations.txt
formatting string and parsing that.

• Scripts/webkitpy/layout_tests/models/test_expectations.py:

(TestExpectationParser):
(TestExpectationParser.parse_skipped_tests):
(TestExpectationParser._parse_line):
(TestExpectationParser._collect_matching_tests):
(TestExpectations.init):
(TestExpectations._add_skipped_tests):

• Scripts/webkitpy/layout_tests/port/base.py:

(Port):
(Port.test_isfile):
(Port.normalize_test_name):
(Port.layout_tests_dir):
(Port.abspath_for_test):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/models/test_expectations.py (modified) (6 diffs)
• Scripts/webkitpy/layout_tests/port/base.py (modified) (4 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-01-25  Ojan Vafai  <ojan@chromium.org>
+
+        Parsing test_expecations.txt + Skipped lists takes too long
+        https://bugs.webkit.org/show_bug.cgi?id=77059
+
+        Reviewed by Dirk Pranke.
+
+        This saves ~100ms on the Apple Mac port.
+        -memoize a bunch of path methods.
+        -Avoid doing multiple disk accesses per line.
+        -Parse the skipped list directly instead of turning it into a test_expecations.txt
+        formatting string and parsing that.
+
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+        (TestExpectationParser):
+        (TestExpectationParser.parse_skipped_tests):
+        (TestExpectationParser._parse_line):
+        (TestExpectationParser._collect_matching_tests):
+        (TestExpectations.__init__):
+        (TestExpectations._add_skipped_tests):
+        * Scripts/webkitpy/layout_tests/port/base.py:
+        (Port):
+        (Port.test_isfile):
+        (Port.normalize_test_name):
+        (Port.layout_tests_dir):
+        (Port.abspath_for_test):
+
 2012-01-25  Ojan Vafai  <ojan@chromium.org>
 

trunk/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

@@ -188 +188 @@
     BUG_MODIFIER_REGEX = 'bug\d+'
     REBASELINE_MODIFIER = 'rebaseline'
+    FAIL_EXPECTATION = 'fail'
     SKIP_MODIFIER = 'skip'
     SLOW_MODIFIER = 'slow'
@@ -206 +207 @@
         return expectations
 
+    def expectation_for_skipped_test(self, test_name):
+        expectation_line = TestExpectationLine()
+        expectation_line.original_string = test_name
+        expectation_line.modifiers = [TestExpectationParser.SKIP_MODIFIER]
+        expectation_line.name = test_name
+        expectation_line.expectations = [TestExpectationParser.FAIL_EXPECTATION]
+        self._parse_line(expectation_line)
+        return expectation_line
+
     def _parse_line(self, expectation_line):
         if not expectation_line.name:
@@ -211 +221 @@
 
         self._check_modifiers_against_expectations(expectation_line)
-        if self._check_path_does_not_exist(expectation_line):
+
+        expectation_line.is_file = self._port.test_isfile(expectation_line.name)
+        if not expectation_line.is_file and self._check_path_does_not_exist(expectation_line):
             return
 
-        expectation_line.path = self._port.normalize_test_name(expectation_line.name)
+        if expectation_line.is_file:
+            expectation_line.path = expectation_line.name
+        else:
+            expectation_line.path = self._port.normalize_test_name(expectation_line.name)
+
         self._collect_matching_tests(expectation_line)
 
@@ -286 +302 @@
             return
 
-        if self._port.test_isdir(expectation_line.path):
+        if not expectation_line.is_file:
             # this is a test category, return all the tests of the category.
             expectation_line.matching_tests = [test for test in self._full_test_list if test.startswith(expectation_line.path)]
@@ -703 +719 @@
         self._parser = TestExpectationParser(port, tests, is_lint_mode)
         self._port = port
-        self._test_configuration_converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros())
         self._skipped_tests_warnings = []
 
@@ -836 +851 @@
             if test.name and test.name in tests_to_skip:
                 self._skipped_tests_warnings.append(':%d %s is also in a Skipped file.' % (index, test.name))
-        skipped_tests = '\n'.join(map(lambda test_path: 'BUG_SKIPPED SKIP : %s = FAIL' % test_path, tests_to_skip))
-        for test in self._parser.parse(skipped_tests):
-            self._model.add_expectation_line(test, overrides_allowed=True)
+        for test_name in tests_to_skip:
+            self._model.add_expectation_line(self._parser.expectation_for_skipped_test(test_name), overrides_allowed=True)

trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py

@@ -519 +519 @@
                       self._filesystem.listdir(layout_tests_dir))
 
+    @memoized
+    def test_isfile(self, test_name):
+        """Return True if the test name refers to a directory of tests."""
+        # Used by test_expectations.py to apply rules to whole directories.
+        test_path = self.abspath_for_test(test_name)
+        return self._filesystem.isfile(test_path)
+
+    @memoized
     def test_isdir(self, test_name):
         """Return True if the test name refers to a directory of tests."""
@@ -541 +549 @@
     def normalize_test_name(self, test_name):
         """Returns a normalized version of the test name or test directory."""
-        if self.test_isdir(test_name) and not test_name.endswith('/'):
+        if test_name.endswith('/'):
+            return test_name
+        if self.test_isdir(test_name):
             return test_name + '/'
         return test_name
@@ -561 +571 @@
         self._filesystem.write_binary_file(baseline_path, data)
 
+    @memoized
     def layout_tests_dir(self):
         """Return the absolute path to the top of the LayoutTests directory."""
-        return self.path_from_webkit_base('LayoutTests')
+        return self._filesystem.normpath(self.path_from_webkit_base('LayoutTests'))
 
     def perf_tests_dir(self):
@@ -698 +709 @@
         return filename[len(self.perf_tests_dir()) + 1:]
 
+    @memoized
     def abspath_for_test(self, test_name):
         """Returns the full path to the file for a given test name. This is the
         inverse of relative_test_filename()."""
-        return self._filesystem.normpath(self._filesystem.join(self.layout_tests_dir(), test_name))
+        return self._filesystem.join(self.layout_tests_dir(), test_name)
 
     def results_directory(self):