Changeset 92153 in webkit

Timestamp:

Aug 1, 2011 5:07:14 PM (13 years ago)

Author:

abarth@webkit.org

Message:

webkit-patch needs to be able to "optimize" the storage of baselines on disk
​https://bugs.webkit.org/show_bug.cgi?id=65418

Reviewed by Dimitri Glazkov.

If we're not careful when rebaselining tests, we can end up with lots
of duplicate expected results files in the tree. This patch adds a
webkit-patch command that optimizes the storage of expected results on
disk.

This command is similar to deduplicate-tests, except that it can move
test results around rather than just remove duplicate results.

Unfortunately, this problem is very tricky because the baseline search
structure is a hypergraph. This patch include a huerstic optimizer
that appears to work on a bunch of examples I've tried. We'll likely
need to refine it as gain experience.

• Scripts/webkitpy/common/system/filesystem.py:
• Scripts/webkitpy/tool/commands/rebaseline.py:

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/common/checkout/baselineoptimizer.py (added)
• Scripts/webkitpy/common/checkout/baselineoptimizer_unittest.py (added)
• Scripts/webkitpy/common/system/filesystem.py (modified) (2 diffs)
• Scripts/webkitpy/common/system/filesystem_mock.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/port/base.py (modified) (1 diff)
• Scripts/webkitpy/tool/commands/rebaseline.py (modified) (2 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-08-01  Adam Barth  <abarth@webkit.org>
+
+        webkit-patch needs to be able to "optimize" the storage of baselines on disk
+        https://bugs.webkit.org/show_bug.cgi?id=65418
+
+        Reviewed by Dimitri Glazkov.
+
+        If we're not careful when rebaselining tests, we can end up with lots
+        of duplicate expected results files in the tree.  This patch adds a
+        webkit-patch command that optimizes the storage of expected results on
+        disk.
+
+        This command is similar to deduplicate-tests, except that it can move
+        test results around rather than just remove duplicate results.
+
+        Unfortunately, this problem is very tricky because the baseline search
+        structure is a hypergraph.  This patch include a huerstic optimizer
+        that appears to work on a bunch of examples I've tried.  We'll likely
+        need to refine it as gain experience.
+
+        * Scripts/webkitpy/common/system/filesystem.py:
+        * Scripts/webkitpy/tool/commands/rebaseline.py:
+
 2011-08-01  Dimitri Glazkov  <dglazkov@chromium.org>
 

trunk/Tools/Scripts/webkitpy/common/system/filesystem.py

@@ -35 +35 @@
 import exceptions
 import glob
+import hashlib
 import os
 import shutil
@@ -223 +224 @@
         with codecs.open(path, 'w', 'utf8') as f:
             f.write(contents)
+
+    def sha1(self, path):
+        contents = self.read_binary_file(path)
+        return hashlib.sha1(contents).hexdigest()
 
     def relpath(self, path, start='.'):

trunk/Tools/Scripts/webkitpy/common/system/filesystem_mock.py

@@ -28 +28 @@
 
 import errno
+import hashlib
 import os
 import re
@@ -285 +286 @@
         return self.write_binary_file(path, contents.encode('utf-8'))
 
+    def sha1(self, path):
+        contents = self.read_binary_file(path)
+        return hashlib.sha1(contents).hexdigest()
+
     def relpath(self, path, start='.'):
         return ospath.relpath(path, start, self.abspath, self.sep)

trunk/Tools/Scripts/webkitpy/layout_tests/port/base.py

@@ -542 +542 @@
         return self.path_from_webkit_base('LayoutTests')
 
+    def webkit_base(self):
+        return self._filesystem.abspath(self.path_from_webkit_base('.'))
+
     def skipped_layout_tests(self):
         return []

trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py

@@ -33 +33 @@
 
 import webkitpy.common.config.urls as config_urls
+from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer
 from webkitpy.common.net.buildbot import BuildBot
 from webkitpy.common.net.layouttestresults import LayoutTestResults
 from webkitpy.common.system.user import User
+from webkitpy.layout_tests.layout_package.test_result_writer import TestResultWriter
 from webkitpy.layout_tests.models import test_failures
 from webkitpy.layout_tests.port import factory
@@ -129 +131 @@
 
 
+class OptimizeBaselines(AbstractDeclarativeCommand):
+    name = "optimize-baselines"
+    help_text = "Reshuffles the baselines for a the given test to use as litte space on disk as possible."
+    argument_names = "TEST_NAME"
+
+    # FIXME: Should TestResultWriter know how to compute this string?
+    def _baseline_name(self, test_name, suffix):
+        return self._tool.filesystem.splitext(test_name)[0] + TestResultWriter.FILENAME_SUFFIX_EXPECTED + suffix
+
+    def execute(self, options, args, tool):
+        baseline_optimizer = BaselineOptimizer(tool.scm(), tool.filesystem)
+
+        test_name = args[0]
+        for suffix in ['.png', '.txt']:
+            baseline_name = self._baseline_name(test_name, suffix)
+            if not baseline_optimizer.optimize(baseline_name):
+                print "Hueristics failed to optimize %s" % baseline_name
+
+
 class Rebaseline(AbstractDeclarativeCommand):
     name = "rebaseline"