Changeset 57399 in webkit

Timestamp:

Apr 9, 2010 7:17:44 PM (14 years ago)

Author:

abarth@webkit.org

Message:

2010-04-09 Adam Barth <​abarth@webkit.org>

Reviewed by Eric Seidel.

Don't reinvent Executive.cpu_count for every port
​https://bugs.webkit.org/show_bug.cgi?id=37377

mac.py and chromium_mac.py had some copy/paste code. This code doesn't
actually have anything to do with WebKit ports. It's really just
something in the multiprocessing package. The lame bit is that package
isn't available in older versions of Python, so we need to implement a
fallback. However, we already have the fallback in common. We don't
need to reinvent it specificly for layout_tests.

• Scripts/webkitpy/common/system/executive.py:
• Scripts/webkitpy/layout_tests/port/base.py:
• Scripts/webkitpy/layout_tests/port/chromium.py:
• Scripts/webkitpy/layout_tests/port/chromium_linux.py:
• Scripts/webkitpy/layout_tests/port/chromium_mac.py:
• Scripts/webkitpy/layout_tests/port/chromium_win.py:
• Scripts/webkitpy/layout_tests/port/mac.py:
• Scripts/webkitpy/layout_tests/port/test.py:
• Scripts/webkitpy/layout_tests/run_webkit_tests.py:

Location:

trunk/WebKitTools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/common/system/executive.py (modified) (2 diffs)
• Scripts/webkitpy/layout_tests/port/base.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/chromium.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/chromium_linux.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/chromium_mac.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/chromium_win.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/mac.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/port/test.py (modified) (1 diff)
• Scripts/webkitpy/layout_tests/run_webkit_tests.py (modified) (5 diffs)

trunk/WebKitTools/ChangeLog

@@ +1 @@
+2010-04-09  Adam Barth  <abarth@webkit.org>
+
+        Reviewed by Eric Seidel.
+
+        Don't reinvent Executive.cpu_count for every port
+        https://bugs.webkit.org/show_bug.cgi?id=37377
+
+        mac.py and chromium_mac.py had some copy/paste code.  This code doesn't
+        actually have anything to do with WebKit ports.  It's really just
+        something in the multiprocessing package.  The lame bit is that package
+        isn't available in older versions of Python, so we need to implement a
+        fallback.  However, we already have the fallback in common.  We don't
+        need to reinvent it specificly for layout_tests.
+
+        * Scripts/webkitpy/common/system/executive.py:
+        * Scripts/webkitpy/layout_tests/port/base.py:
+        * Scripts/webkitpy/layout_tests/port/chromium.py:
+        * Scripts/webkitpy/layout_tests/port/chromium_linux.py:
+        * Scripts/webkitpy/layout_tests/port/chromium_mac.py:
+        * Scripts/webkitpy/layout_tests/port/chromium_win.py:
+        * Scripts/webkitpy/layout_tests/port/mac.py:
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+
 2010-04-09  Adam Barth  <abarth@webkit.org>
 

trunk/WebKitTools/Scripts/webkitpy/common/system/executive.py

@@ -35 +35 @@
 
 import os
+import platform
 import StringIO
 import subprocess
@@ -123 +124 @@
         if multiprocessing:
             return multiprocessing.cpu_count()
+        # Darn.  We don't have the multiprocessing package.
+        system_name = platform.system()
+        if system_name == "Dawin":
+            return int(self.run_command(["sysctl", "-n", "hw.ncpu"]))
+        elif system_name == "Windows":
+            return int(os.environ.get('NUMBER_OF_PROCESSORS', 1))
+        elif system_name == "Linux":
+            num_cores = os.sysconf("SC_NPROCESSORS_ONLN")
+            if isinstance(num_cores, int) and num_cores > 0:
+                return num_cores
         # This quantity is a lie but probably a reasonable guess for modern
         # machines.

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py

@@ -286 +286 @@
         may be different (e.g., 'win-xp' instead of 'chromium-win-xp'."""
         return self._name
-
-    def num_cores(self):
-        """Return the number of cores/cpus available on this machine.
-
-        This routine is used to determine the default amount of parallelism
-        used by run-chromium-webkit-tests."""
-        raise NotImplementedError('Port.num_cores')
 
     # FIXME: This could be replaced by functions in webkitpy.common.checkout.scm.

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py

@@ -45 +45 @@
 
 
+# FIXME: This function doesn't belong in this package.
 def check_file_exists(path_to_file, file_description, override_step=None,
                       logging=True):

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py

@@ -74 +74 @@
         return result
 
-    def num_cores(self):
-        num_cores = os.sysconf("SC_NPROCESSORS_ONLN")
-        if isinstance(num_cores, int) and num_cores > 0:
-            return num_cores
-        return 1
-
     def test_platform_name(self):
         # We use 'linux' instead of 'chromium-linux' in test_expectations.txt.

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py

@@ -66 +66 @@
                        'MacBuildInstructions')
         return result
-
-    def num_cores(self):
-        return int(subprocess.Popen(['sysctl','-n','hw.ncpu'],
-                                    stdout=subprocess.PIPE).stdout.read())
 
     def test_platform_name(self):

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py

@@ -78 +78 @@
         abspath = os.path.abspath(filename)
         return abspath.replace('\\', '/')
-
-    def num_cores(self):
-        return int(os.environ.get('NUMBER_OF_PROCESSORS', 1))
 
     def relative_test_filename(self, filename):

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py

@@ -173 +173 @@
         return result
 
-    def num_cores(self):
-        ncores = int(executive.run_command(["sysctl", "-n", "hw.ncpu"]))
-        # FIXME: new-run-webkit-tests is unstable running more than four
-        # threads in parallel.
-        # See https://bugs.webkit.org/show_bug.cgi?id=36622
-        if ncores > 4:
-            ncores = 4
-        return ncores
-
     def path_to_test_expectations_file(self):
         return self.path_from_webkit_base('LayoutTests', 'platform',

trunk/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py

@@ -70 +70 @@
         return self._name
 
-    def num_cores(self):
-        return int(os.popen2("sysctl -n hw.ncpu")[1].read())
-
     def options(self):
         return self._options

trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -51 +51 @@
 import optparse
 import os
+import platform
 import Queue
 import random
@@ -71 +72 @@
 from test_types import test_type_base
 from test_types import text_diff
+
+from webkitpy.common.system.executive import Executive
 
 import port
@@ -1390 +1393 @@
 
     port_obj = port.get(options.platform, options)
+    executive = Executive()
 
     if not options.configuration:
@@ -1423 +1427 @@
 
     if not options.num_dump_render_trees:
-        # TODO(ojan): Investigate perf/flakiness impact of using numcores + 1.
-        options.num_dump_render_trees = port_obj.num_cores()
+        # FIXME: Investigate perf/flakiness impact of using cpu_count + 1.
+        options.num_dump_render_trees = executive.cpu_count()
+        # FIXME: new-run-webkit-tests is unstable on Mac running more than
+        # four threads in parallel.
+        # See https://bugs.webkit.org/show_bug.cgi?id=36622
+        if platform.system() == "Dawin" and options.num_dump_render_trees > 4:
+            options.num_dump_render_trees = 4
 
     write = create_logging_writer(options, 'config')
@@ -1462 +1471 @@
         # all the test list parsing and ensures it's correct syntax (e.g. no
         # dupes).
-        for platform in port_obj.test_platform_names():
-            test_runner.parse_expectations(platform, is_debug_mode=True)
-            test_runner.parse_expectations(platform, is_debug_mode=False)
+        for platform_name in port_obj.test_platform_names():
+            test_runner.parse_expectations(platform_name, is_debug_mode=True)
+            test_runner.parse_expectations(platform_name, is_debug_mode=False)
         meter.update("")
         print ("If there are no fail messages, errors or exceptions, then the "