Changeset 121699 in webkit

Timestamp:

Jul 2, 2012 12:22:43 PM (12 years ago)

Author:

ojan@chromium.org

Message:

Move rebaseline-all command from the gardening-server down into webkit-patch
​https://bugs.webkit.org/show_bug.cgi?id=90395

Reviewed by Adam Barth.

This is just moving code. It it in preparation for making rebaseline-expectations
use the same code in order to get the parallelism benefits and reduces the amount
of code we have for doing rebaselines.

• Scripts/webkitpy/common/checkout/checkout_unittest.py:

(CheckoutTest.test_apply_patch):
Updated due to the change to executive_mock.

• Scripts/webkitpy/common/system/executive_mock.py:

(MockExecutive.run_command):
Update to print out the input passed to stdin.

• Scripts/webkitpy/tool/commands/download_unittest.py:

Updated due to executive_mock change.

• Scripts/webkitpy/tool/commands/rebaseline.py:

(RebaselineAll):
(RebaselineAll._run_webkit_patch):
(RebaselineAll._builders_to_fetch_from):
(RebaselineAll._rebaseline_commands):
(RebaselineAll._files_to_add):
(RebaselineAll._optimize_baselines):
(RebaselineAll._rebaseline):
(RebaselineAll.execute):
All this code is just copy-pasted except for mechanical changes
(e.g. self.server.tool --> self._tool) and the reading in of the
JSON from stdin instead of the post body.

• Scripts/webkitpy/tool/commands/rebaseline_unittest.py:

(test_rebaseline_all):
Copied the test-case out of gardeningserver_unittest.py.

• Scripts/webkitpy/tool/servers/gardeningserver.py:

(GardeningHTTPRequestHandler):
(GardeningHTTPRequestHandler.rebaseline):
(GardeningHTTPRequestHandler.rebaselineall):

• Scripts/webkitpy/tool/servers/gardeningserver_unittest.py:

(GardeningServerTest.test_rebaselineall):
(GardeningServerTest.test_rebaselineall.run_command):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/common/checkout/checkout_unittest.py (modified) (1 diff)
• Scripts/webkitpy/common/system/executive_mock.py (modified) (1 diff)
• Scripts/webkitpy/tool/commands/download_unittest.py (modified) (1 diff)
• Scripts/webkitpy/tool/commands/rebaseline.py (modified) (2 diffs)
• Scripts/webkitpy/tool/commands/rebaseline_unittest.py (modified) (1 diff)
• Scripts/webkitpy/tool/servers/gardeningserver.py (modified) (2 diffs)
• Scripts/webkitpy/tool/servers/gardeningserver_unittest.py (modified) (3 diffs)
• Scripts/webkitpy/tool/servers/reflectionhandler.py (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-07-02  Ojan Vafai  <ojan@chromium.org>
+
+        Move rebaseline-all command from the gardening-server down into webkit-patch
+        https://bugs.webkit.org/show_bug.cgi?id=90395
+
+        Reviewed by Adam Barth.
+
+        This is just moving code. It it in preparation for making rebaseline-expectations
+        use the same code in order to get the parallelism benefits and reduces the amount
+        of code we have for doing rebaselines.
+
+        * Scripts/webkitpy/common/checkout/checkout_unittest.py:
+        (CheckoutTest.test_apply_patch):
+        Updated due to the change to executive_mock.
+        * Scripts/webkitpy/common/system/executive_mock.py:
+        (MockExecutive.run_command):
+        Update to print out the input passed to stdin.
+        * Scripts/webkitpy/tool/commands/download_unittest.py:
+        Updated due to executive_mock change.
+        * Scripts/webkitpy/tool/commands/rebaseline.py:
+        (RebaselineAll):
+        (RebaselineAll._run_webkit_patch):
+        (RebaselineAll._builders_to_fetch_from):
+        (RebaselineAll._rebaseline_commands):
+        (RebaselineAll._files_to_add):
+        (RebaselineAll._optimize_baselines):
+        (RebaselineAll._rebaseline):
+        (RebaselineAll.execute):
+        All this code is just copy-pasted except for mechanical changes
+        (e.g. self.server.tool --> self._tool) and the reading in of the 
+        JSON from stdin instead of the post body.
+        * Scripts/webkitpy/tool/commands/rebaseline_unittest.py:
+        (test_rebaseline_all):
+        Copied the test-case out of gardeningserver_unittest.py.
+        * Scripts/webkitpy/tool/servers/gardeningserver.py:
+        (GardeningHTTPRequestHandler):
+        (GardeningHTTPRequestHandler.rebaseline):
+        (GardeningHTTPRequestHandler.rebaselineall):
+        * Scripts/webkitpy/tool/servers/gardeningserver_unittest.py:
+        (GardeningServerTest.test_rebaselineall):
+        (GardeningServerTest.test_rebaselineall.run_command):
+
 2012-07-02  Ojan Vafai  <ojan@chromium.org>
 

trunk/Tools/Scripts/webkitpy/common/checkout/checkout_unittest.py

@@ -260 +260 @@
         mock_patch.contents = lambda: "foo"
         mock_patch.reviewer = lambda: None
-        expected_stderr = "MOCK run_command: ['svn-apply', '--force'], cwd=/mock-checkout\n"
+        expected_stderr = "MOCK run_command: ['svn-apply', '--force'], cwd=/mock-checkout, input=foo\n"
         OutputCapture().assert_outputs(self, checkout.apply_patch, [mock_patch], expected_stderr=expected_stderr)

trunk/Tools/Scripts/webkitpy/common/system/executive_mock.py

@@ -89 +89 @@
             if env:
                 env_string = ", env=%s" % env
-            log("MOCK run_command: %s, cwd=%s%s" % (args, cwd, env_string))
+            input_string = ""
+            if input:
+                input_string = ", input=%s" % input
+            log("MOCK run_command: %s, cwd=%s%s%s" % (args, cwd, env_string, input_string))
         output = "MOCK output of child process"
         if self._should_throw:

trunk/Tools/Scripts/webkitpy/tool/commands/download_unittest.py

@@ -137 +137 @@
         expected_stderr = """MOCK run_and_throw_if_fail: ['mock-prepare-ChangeLog', '--email=MOCK email', '--merge-base=None', 'MockFile1'], cwd=/mock-checkout
 MOCK run_and_throw_if_fail: ['mock-check-webkit-style', '--git-commit', 'MOCK git commit', '--diff-files', 'MockFile1', '--filter', '-changelog'], cwd=/mock-checkout
-MOCK run_command: ['ruby', '-I', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch/prettify.rb'], cwd=None
+MOCK run_command: ['ruby', '-I', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch', '/mock-checkout/Websites/bugs.webkit.org/PrettyPatch/prettify.rb'], cwd=None, input=Patch1
 MOCK: user.open_url: file://...
 Was that diff correct?

trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline.py

@@ -33 +33 @@
 import re
 import shutil
+import sys
 import urllib
 
@@ -302 +303 @@
 
 
+class RebaselineAll(AbstractDeclarativeCommand):
+    name = "rebaseline-all"
+    help_text = "Rebaseline based off JSON passed to stdin. Intended to only be called from other scripts."
+
+    def _run_webkit_patch(self, args):
+        try:
+            self._tool.executive.run_command([self._tool.path()] + args, cwd=self._tool.scm().checkout_root)
+        except ScriptError, e:
+            _log.error(e)
+
+    def _builders_to_fetch_from(self, builders):
+        # This routine returns the subset of builders that will cover all of the baseline search paths
+        # used in the input list. In particular, if the input list contains both Release and Debug
+        # versions of a configuration, we *only* return the Release version (since we don't save
+        # debug versions of baselines).
+        release_builders = set()
+        debug_builders = set()
+        builders_to_fallback_paths = {}
+        for builder in builders:
+            port = self._tool.port_factory.get_from_builder_name(builder)
+            if port.test_configuration().build_type == 'Release':
+                release_builders.add(builder)
+            else:
+                debug_builders.add(builder)
+        for builder in list(release_builders) + list(debug_builders):
+            port = self._tool.port_factory.get_from_builder_name(builder)
+            fallback_path = port.baseline_search_path()
+            if fallback_path not in builders_to_fallback_paths.values():
+                builders_to_fallback_paths[builder] = fallback_path
+        return builders_to_fallback_paths.keys()
+
+    def _rebaseline_commands(self, test_list):
+        path_to_webkit_patch = self._tool.path()
+        cwd = self._tool.scm().checkout_root
+        commands = []
+        for test in test_list:
+            for builder in self._builders_to_fetch_from(test_list[test]):
+                suffixes = ','.join(test_list[test][builder])
+                cmd_line = [path_to_webkit_patch, 'rebaseline-test', '--print-scm-changes', '--suffixes', suffixes, builder, test]
+                commands.append(tuple([cmd_line, cwd]))
+        return commands
+
+    def _files_to_add(self, command_results):
+        files_to_add = set()
+        for output in [result[1] for result in command_results]:
+            try:
+                files_to_add.update(json.loads(output)['add'])
+            except ValueError, e:
+                _log.warning('"%s" is not a JSON object, ignoring' % output)
+
+        return list(files_to_add)
+
+    def _optimize_baselines(self, test_list):
+        # We don't run this in parallel because modifying the SCM in parallel is unreliable.
+        for test in test_list:
+            all_suffixes = set()
+            for builder in self._builders_to_fetch_from(test_list[test]):
+                all_suffixes.update(test_list[test][builder])
+            self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(all_suffixes), test])
+
+    def _rebaseline(self, json_input):
+        test_list = json.loads(json_input)
+
+        commands = self._rebaseline_commands(test_list)
+        command_results = self._tool.executive.run_in_parallel(commands)
+
+        files_to_add = self._files_to_add(command_results)
+        self._tool.scm().add_list(list(files_to_add))
+
+        self._optimize_baselines(test_list)
+
+    def execute(self, options, args, tool):
+        self._rebaseline(sys.stdin.read())
+
+
 class Rebaseline(AbstractDeclarativeCommand):
     name = "rebaseline"

trunk/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py

@@ -190 +190 @@
         OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs)
 
+    def test_rebaseline_all(self):
+        old_exact_matches = builders._exact_matches
+        builders._exact_matches = {
+            "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
+            "MOCK builder (Debug)": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier", "debug"])},
+        }
+
+        command = RebaselineAll()
+        tool = MockTool()
+        command.bind_to_tool(tool)
+        tool.executive = MockExecutive(should_log=True)
+
+        expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test', '--print-scm-changes', '--suffixes', u'txt,png', u'MOCK builder', u'user-scripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', u'txt,png', u'user-scripts/another-test.html'], cwd=/mock-checkout
+"""
+        OutputCapture().assert_outputs(self, command._rebaseline, ['{"user-scripts/another-test.html":{"MOCK builder": ["txt","png"]}}'], expected_stderr=expected_stderr)
+
+        expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test', '--print-scm-changes', '--suffixes', u'txt,png', u'MOCK builder (Debug)', u'user-scripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', u'txt,png', u'user-scripts/another-test.html'], cwd=/mock-checkout
+"""
+        OutputCapture().assert_outputs(self, command._rebaseline, ['{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"]}}'], expected_stderr=expected_stderr)
+
+        expected_stderr = """MOCK run_command: ['echo', 'rebaseline-test', '--print-scm-changes', '--suffixes', u'txt', u'MOCK builder', u'user-scripts/another-test.html'], cwd=/mock-checkout
+MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', u'txt', u'user-scripts/another-test.html'], cwd=/mock-checkout
+"""
+        OutputCapture().assert_outputs(self, command._rebaseline, ['{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"], "MOCK builder": ["txt"]}}'], expected_stderr=expected_stderr)
+
+        builders._exact_matches = old_exact_matches
+
     def test_rebaseline_expectations(self):
         command = RebaselineExpectations()

trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver.py

@@ -141 +141 @@
         self._serve_text('success')
 
+    # FIXME: Is this dead code?
     def rebaseline(self):
         builder = self.query['builder'][0]
@@ -156 +157 @@
         self._serve_text('success')
 
-    def _builders_to_fetch_from(self, builders):
-        # This routine returns the subset of builders that will cover all of the baseline search paths
-        # used in the input list. In particular, if the input list contains both Release and Debug
-        # versions of a configuration, we *only* return the Release version (since we don't save
-        # debug versions of baselines).
-        release_builders = set()
-        debug_builders = set()
-        builders_to_fallback_paths = {}
-        for builder in builders:
-            port = self.server.tool.port_factory.get_from_builder_name(builder)
-            if port.test_configuration().build_type == 'Release':
-                release_builders.add(builder)
-            else:
-                debug_builders.add(builder)
-        for builder in list(release_builders) + list(debug_builders):
-            port = self.server.tool.port_factory.get_from_builder_name(builder)
-            fallback_path = port.baseline_search_path()
-            if fallback_path not in builders_to_fallback_paths.values():
-                builders_to_fallback_paths[builder] = fallback_path
-        return builders_to_fallback_paths.keys()
-
-    def _rebaseline_commands(self, test_list):
-        path_to_webkit_patch = self.server.tool.path()
-        cwd = self.server.tool.scm().checkout_root
-        commands = []
-        for test in test_list:
-            for builder in self._builders_to_fetch_from(test_list[test]):
-                suffixes = ','.join(test_list[test][builder])
-                cmd_line = [path_to_webkit_patch, 'rebaseline-test', '--print-scm-changes', '--suffixes', suffixes, builder, test]
-                commands.append(tuple([cmd_line, cwd]))
-        return commands
-
-    def _files_to_add(self, command_results):
-        files_to_add = set()
-        for output in [result[1] for result in command_results]:
-            try:
-                files_to_add.update(json.loads(output)['add'])
-            except ValueError, e:
-                _log.warning('"%s" is not a JSON object, ignoring' % output)
-
-        return list(files_to_add)
-
-    def _optimize_baselines(self, test_list):
-        # We don't run this in parallel because modifying the SCM in parallel is unreliable.
-        for test in test_list:
-            all_suffixes = set()
-            for builder in self._builders_to_fetch_from(test_list[test]):
-                all_suffixes.update(test_list[test][builder])
-            self._run_webkit_patch(['optimize-baselines', '--suffixes', ','.join(all_suffixes), test])
-
     def rebaselineall(self):
-        test_list = self._read_entity_body_as_json()
-
-        commands = self._rebaseline_commands(test_list)
-        command_results = self.server.tool.executive.run_in_parallel(commands)
-
-        files_to_add = self._files_to_add(command_results)
-        self.server.tool.scm().add_list(list(files_to_add))
-
-        self._optimize_baselines(test_list)
+        command = ['rebaseline-all']
+        self.server.tool.executive.run_command([self.server.tool.path()] + command, input=self.read_entity_body(), cwd=self.server.tool.scm().checkout_root)
         self._serve_text('success')

trunk/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py

@@ -70 +70 @@
         return GardeningExpectationsUpdater(self.server.tool, TestPortFactory.create())
 
-    def _read_entity_body(self):
+    def read_entity_body(self):
         return self.body if self.body else ''
 
@@ -184 +184 @@
 
     def test_rebaselineall(self):
-        builders._exact_matches = {
-            "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])},
-            "MOCK builder (Debug)": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier", "debug"])},
-        }
-        expected_stderr = "MOCK run_command: ['echo', 'rebaseline-test', '--print-scm-changes', '--suffixes', u'%s', u'%s', u'user-scripts/another-test.html'], cwd=/mock-checkout\nMOCK run_command: ['echo', 'optimize-baselines', '--suffixes', u'%s', u'user-scripts/another-test.html'], cwd=/mock-checkout\n"
+        expected_stderr = "MOCK run_command: ['echo', 'rebaseline-all'], cwd=/mock-checkout, input={\"user-scripts/another-test.html\":{\"%s\": [%s]}}\n"
         expected_stdout = "== Begin Response ==\nsuccess\n== End Response ==\n"
         server = MockServer()
@@ -194 +190 @@
         self.output = ['{"add": [], "delete": []}', '']
 
-        def run_command(args, cwd=None, **kwargs):
-            print >> sys.stderr, "MOCK run_command: %s, cwd=%s" % (args, cwd)
+        def run_command(args, cwd=None, input=None, **kwargs):
+            print >> sys.stderr, "MOCK run_command: %s, cwd=%s, input=%s" % (args, cwd, input)
             return self.output.pop(0)
 
         server.tool.executive.run_command = run_command
-        self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder": ["txt","png"]}}', expected_stderr=expected_stderr % ('txt,png', 'MOCK builder', 'txt,png'), expected_stdout=expected_stdout, server=server)
-
-        self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"]}}', expected_stderr=expected_stderr % ('txt,png', 'MOCK builder (Debug)', 'txt,png'), expected_stdout=expected_stdout)
-
-        self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"], "MOCK builder": ["txt"]}}', expected_stderr=expected_stderr % ('txt', 'MOCK builder', 'txt'), expected_stdout=expected_stdout)
+        self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder": ["txt","png"]}}', expected_stderr=expected_stderr % ('MOCK builder', '"txt","png"'), expected_stdout=expected_stdout, server=server)
+
+        self._post_to_path("/rebaselineall", body='{"user-scripts/another-test.html":{"MOCK builder (Debug)": ["txt","png"]}}', expected_stderr=expected_stderr % ('MOCK builder (Debug)', '"txt","png"'), expected_stdout=expected_stdout)
 
     def test_rebaseline_new_port(self):

trunk/Tools/Scripts/webkitpy/tool/servers/reflectionhandler.py

@@ -60 +60 @@
         self._handle_request()
 
-    def _read_entity_body(self):
+    def read_entity_body(self):
         length = int(self.headers.getheader('content-length'))
         return self.rfile.read(length)
 
     def _read_entity_body_as_json(self):
-        return json.loads(self._read_entity_body())
+        return json.loads(self.read_entity_body())
 
     def _handle_request(self):