Changeset 90770 in webkit

Timestamp:

Jul 11, 2011, 11:51:13 AM (14 years ago)

Author:

abarth@webkit.org

Message:

Add a webkit-patch command for rebaselining an individual test
​https://bugs.webkit.org/show_bug.cgi?id=64246

Reviewed by Eric Seidel.

This patch introduces a command that's able to rebaseline a single
test. Currently, the command works only with the build.chromium.org
buildbots, but extending it to work with the build.webkit.org bots
shouldn't be that hard.

A complete rebaseling tool should also include an "optimize baselines"
command (which moves/deletes baselines in order to reduce the number of
expected results files), but that will come in a future patch.

Really BuilderToPort should be merged into builders.py, but I'm going
to save that for a future patch as well. (We need to stop shaving yaks
at some point.)

• Scripts/webkitpy/tool/commands/rebaseline.py:

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/common/host.py (modified) (2 diffs)
• Scripts/webkitpy/common/net/buildbot/buildbot.py (modified) (3 diffs)
• Scripts/webkitpy/common/net/buildbot/chromiumbuildbot.py (added)
• Scripts/webkitpy/common/net/web.py (added)
• Scripts/webkitpy/layout_tests/port/builders.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/port/builders_unittest.py (modified) (1 diff)
• Scripts/webkitpy/tool/commands/rebaseline.py (modified) (4 diffs)
• Scripts/webkitpy/tool/commands/rebaseline_unittest.py (modified) (2 diffs)
• Scripts/webkitpy/tool/mocktool.py (modified) (2 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-07-11  Adam Barth  <abarth@webkit.org>
+
+        Add a webkit-patch command for rebaselining an individual test
+        https://bugs.webkit.org/show_bug.cgi?id=64246
+
+        Reviewed by Eric Seidel.
+
+        This patch introduces a command that's able to rebaseline a single
+        test.  Currently, the command works only with the build.chromium.org
+        buildbots, but extending it to work with the build.webkit.org bots
+        shouldn't be that hard.
+
+        A complete rebaseling tool should also include an "optimize baselines"
+        command (which moves/deletes baselines in order to reduce the number of
+        expected results files), but that will come in a future patch.
+
+        Really BuilderToPort should be merged into builders.py, but I'm going
+        to save that for a future patch as well.  (We need to stop shaving yaks
+        at some point.)
+
+        * Scripts/webkitpy/tool/commands/rebaseline.py:
+
 2011-07-11  Adam Barth  <abarth@webkit.org>
 

trunk/Tools/Scripts/webkitpy/common/host.py

@@ -32 +32 @@
 from webkitpy.common.checkout.scm import default_scm
 from webkitpy.common.config.ports import WebKitPort
-from webkitpy.common.net import bugzilla, buildbot, statusserver
+from webkitpy.common.net import bugzilla, buildbot, statusserver, web
 from webkitpy.common.net.irc import ircproxy
 from webkitpy.common.system import executive, filesystem, platforminfo, user, workspace
@@ -43 +43 @@
         self.buildbot = buildbot.BuildBot()
         self.executive = executive.Executive()
+        self.web = web.Web()
         self._irc = None
         self.filesystem = filesystem.FileSystem()

trunk/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py

@@ -69 +69 @@
     def results_url(self):
         return "%s/results/%s" % (self._buildbot.buildbot_url, self.url_encoded_name())
+
+    # In addition to per-build results, the build.chromium.org builders also
+    # keep a directory that accumulates test results over many runs.
+    def accumulated_results_url(self):
+        return None
 
     def url_encoded_name(self):
@@ -281 +286 @@
 
 class BuildBot(object):
-    def __init__(self, url=config_urls.buildbot_url):
-        self.buildbot_url = url
+    _builder_factory = Builder
+    _default_url = config_urls.buildbot_url
+
+    def __init__(self, url=None):
+        self.buildbot_url = url if url else self._default_url
         self._builder_by_name = {}
 
@@ -441 +449 @@
         builder = self._builder_by_name.get(name)
         if not builder:
-            builder = Builder(name, self)
+            builder = self._builder_factory(name, self)
             self._builder_by_name[name] = builder
         return builder

trunk/Tools/Scripts/webkitpy/layout_tests/port/builders.py

@@ -31 +31 @@
 
 
-def _path_from_name(name):
-    return re.sub(r'[\s().]', '_', name)
+def builder_path_from_name(builder_name):
+    return re.sub(r'[\s().]', '_', builder_name)
+
 
 # Compiled manually from http://build.chromium.org/p/chromium/json/builders/help?as_text=1
@@ -39 +40 @@
 # FIXME Make the values in this map into lists.
 CHROMIUM_PORT_TO_BUILDER_NAME = {
-    'chromium-gpu-linux': _path_from_name('Webkit Linux - GPU'),
+    'chromium-gpu-linux': builder_path_from_name('Webkit Linux - GPU'),
 
-    'chromium-gpu-mac-snowleopard': _path_from_name('Webkit Mac10.6 - GPU'),
-    'chromium-gpu-mac-leopard': _path_from_name('Webkit Mac10.5 - GPU'),
+    'chromium-gpu-mac-snowleopard': builder_path_from_name('Webkit Mac10.6 - GPU'),
+    'chromium-gpu-mac-leopard': builder_path_from_name('Webkit Mac10.5 - GPU'),
 
-    'chromium-gpu-win-xp': _path_from_name('Webkit Win - GPU'),
-    'chromium-gpu-win-vista': _path_from_name('Webkit Vista - GPU'),
-    'chromium-gpu-win-win7': _path_from_name('Webkit Win7 - GPU'),
+    'chromium-gpu-win-xp': builder_path_from_name('Webkit Win - GPU'),
+    'chromium-gpu-win-vista': builder_path_from_name('Webkit Vista - GPU'),
+    'chromium-gpu-win-win7': builder_path_from_name('Webkit Win7 - GPU'),
 
-    'chromium-linux-x86_64': _path_from_name('Linux Tests x64'),
-    'chromium-linux-x86': _path_from_name('Linux Tests (dbg)(1)'),
+    'chromium-linux-x86_64': builder_path_from_name('Linux Tests x64'),
+    'chromium-linux-x86': builder_path_from_name('Linux Tests (dbg)(1)'),
 
-    'chromium-mac-leopard': _path_from_name('Mac10.5 Tests (1)'),
-    'chromium-mac-snowleopard': _path_from_name('Mac 10.6 Tests (dbg)(1)'),
+    'chromium-mac-leopard': builder_path_from_name('Mac10.5 Tests (1)'),
+    'chromium-mac-snowleopard': builder_path_from_name('Mac 10.6 Tests (dbg)(1)'),
 
-    'chromium-win-xp': _path_from_name('XP Tests (dbg)(5)'),
-    'chromium-win-vista': _path_from_name('Vista Tests (dbg)(1)'),
+    'chromium-win-xp': builder_path_from_name('XP Tests (dbg)(5)'),
+    'chromium-win-vista': builder_path_from_name('Vista Tests (dbg)(1)'),
     'chromium-win-win7': None,
 

trunk/Tools/Scripts/webkitpy/layout_tests/port/builders_unittest.py

@@ -39 +39 @@
         }
         for name, expected in tests.items():
-            self.assertEquals(expected, builders._path_from_name(name))
+            self.assertEquals(expected, builders.builder_path_from_name(name))
 
 if __name__ == '__main__':

trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py

@@ -32 +32 @@
 import urllib
 
+import webkitpy.common.config.urls as config_urls
 from webkitpy.common.net.buildbot import BuildBot
+from webkitpy.common.net.buildbot.chromiumbuildbot import ChromiumBuildBot
 from webkitpy.common.net.layouttestresults import LayoutTestResults
 from webkitpy.common.system.user import User
@@ -41 +43 @@
 
 
-# FIXME: I'm not sure where this logic should go in the end.
-# For now it's here, until we have a second need for it.
+# FIXME: This logic should be moved to builders.py.
 class BuilderToPort(object):
     _builder_name_to_port_name = {
+        # These builders are on build.webkit.org.
         r"SnowLeopard": "mac-snowleopard",
         r"Leopard": "mac-leopard",
@@ -53 +55 @@
         r"Chromium Linux": "chromium-linux",
         r"Chromium Win": "chromium-win",
+
+        # These builders are on build.chromium.org.
+        r"Webkit Win": "chromium-win-xp",
+        r"Webkit Vista": "chromium-win-vista",
+        r"Webkit Win7": "chromium-win-win7",
+        r"Webkit Win (dbg)(1)": "chromium-win-win7",  # FIXME: Is this correct?
+        r"Webkit Win (dbg)(2)": "chromium-win-win7",  # FIXME: Is this correct?
+        r"Webkit Linux": "chromium-linux-x86_64",
+        r"Webkit Linux 32": "chromium-linux-x86",
+        r"Webkit Linux (dbg)(1)": "chromium-linux-x86_64",
+        r"Webkit Linux (dbg)(2)": "chromium-linux-x86_64",
+        r"Webkit Mac10\.5": "chromium-mac-leopard",
+        r"Webkit Mac10\.5 (dbg)(1)": "chromium-mac-leopard",
+        r"Webkit Mac10\.5 (dbg)(2)": "chromium-mac-leopard",
+        r"Webkit Mac10\.6": "chromium-mac-snowleopard",
+        r"Webkit Mac10\.6 (dbg)": "chromium-mac-snowleopard",
     }
 
@@ -66 +84 @@
         assert(port)  # Need to update _builder_name_to_port_name
         return port
+
+
+class RebaselineTest(AbstractDeclarativeCommand):
+    name = "rebaseline-test"
+    help_text = "Rebaseline a single test from a buildbot.  (Currently works only with build.chromium.org buildbots.)"
+    argument_names = "BUILDER_NAME TEST_NAME SUFFIX"
+
+    def _results_url(self, builder_name):
+        # FIXME: Generalize this command to work with non-build.chromium.org builders.
+        # FIXME: We should really get the buildbot from the tool!
+        builder = ChromiumBuildBot().builder_with_name(builder_name)
+        return builder.accumulated_results_url()
+
+    def _baseline_directory(self, builder_name):
+        port = BuilderToPort().port_for_builder(builder_name)
+        return port.baseline_path()
+
+    def _save_baseline(self, data, target_baseline):
+        self._tool.filesystem.write_binary_file(target_baseline, data)
+        if not self._tool.scm().exists(target_baseline):
+            self._tool.scm().add(target_baseline)
+
+    def _test_root(self, test_name):
+        return os.path.splitext(test_name)[0]
+
+    def _file_name_for_actual_result(self, test_name, suffix):
+        return "%s-actual.%s" % (self._test_root(test_name), suffix)
+
+    def _file_name_for_expected_result(self, test_name, suffix):
+        return "%s-expected.%s" % (self._test_root(test_name), suffix)
+
+    def _rebaseline_test(self, builder_name, test_name, suffix):
+        results_url = self._results_url(builder_name)
+        baseline_directory = self._baseline_directory(builder_name)
+
+        source_baseline = "%s/%s" % (results_url, self._file_name_for_actual_result(test_name, suffix))
+        target_baseline = os.path.join(baseline_directory, self._file_name_for_expected_result(test_name, suffix))
+
+        print "Retrieving %s ..." % source_baseline
+        self._save_baseline(self._tool.web.get_binary(source_baseline), target_baseline)
+
+    def execute(self, options, args, tool):
+        self._rebaseline_test(args[0], args[1], args[2])
 
 

trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py

@@ -31 +31 @@
 from webkitpy.common.system.outputcapture import OutputCapture
 from webkitpy.thirdparty.mock import Mock
-from webkitpy.tool.commands.rebaseline import BuilderToPort, Rebaseline
+from webkitpy.tool.commands.rebaseline import *
 from webkitpy.tool.mocktool import MockTool
 
 
-class RebaselineTest(unittest.TestCase):
+class TestRebaseline(unittest.TestCase):
     # This just makes sure the code runs without exceptions.
     def test_tests_to_update(self):
@@ -44 +44 @@
 
 
+class TestRebaselineTest(unittest.TestCase):
+    def test_tests_to_update(self):
+        command = RebaselineTest()
+        command.bind_to_tool(MockTool())
+        build = Mock()
+        expected_stdout = "Retrieving http://build.chromium.org/f/chromium/layout_test_results/Webkit_Linux/results/layout-test-results/userscripts/another-test-actual.txt ...\n"
+        OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Linux", "userscripts/another-test.html", "txt"], expected_stdout=expected_stdout)
+
+
 class BuilderToPortTest(unittest.TestCase):
     def test_port_for_builder(self):

trunk/Tools/Scripts/webkitpy/tool/mocktool.py

@@ -748 +748 @@
 
 
+class MockWeb(object):
+    def get_binary(self, url):
+        return "MOCK Web result"
+
+
 class MockTool(object):
 
@@ -755 +760 @@
         self.buildbot = MockBuildBot()
         self.executive = MockExecutive(should_log=log_executive)
+        self.web = MockWeb()
         self.filesystem = MockFileSystem()
         self.workspace = MockWorkspace()