Changeset 219853 in webkit

Timestamp:

Jul 24, 2017 6:42:03 PM (7 years ago)

Author:

commit-queue@webkit.org

Message:

Add functions to autoinstaller needed for Benchmark Runner script
​https://bugs.webkit.org/show_bug.cgi?id=174331

Patch by Matthew Stewart <​matthew_r_stewart@apple.com> on 2017-07-24
Reviewed by Dean Johnson, Dewei Zhu, Stephanie Lewis.

Adds autoinstaller functions to install selenium and webdriver binaries
which will be used by the browser driver part of Benchmark Runner.

• Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py:

(BrowserDriver.restore_env):
(BrowserDriver):
(BrowserDriver.get_webdriver_binary_path):

• Scripts/webkitpy/benchmark_runner/browser_driver/linux_chrome_driver.py:

(LinuxChromeDriver.launch_driver):

• Scripts/webkitpy/benchmark_runner/browser_driver/linux_firefox_driver.py:

(LinuxFirefoxDriver.launch_driver):

• Scripts/webkitpy/benchmark_runner/browser_driver/osx_chrome_driver.py:

(OSXChromeDriver.launch_driver):
(OSXChromeCanaryDriver.launch_driver):

• Scripts/webkitpy/benchmark_runner/browser_driver/osx_firefox_driver.py:

(OSXFirefoxDriver.launch_driver):
(OSXFirefoxNightlyDriver.launch_driver):

• Scripts/webkitpy/benchmark_runner/browser_driver/osx_safari_driver.py:
• Scripts/webkitpy/benchmark_runner/utils.py:

(get_driver_binary_path):

• Scripts/webkitpy/common/system/autoinstall.py:

(AutoInstaller._unzip):

• Scripts/webkitpy/thirdparty/init.py:

(AutoinstallImportHook.find_module):
(AutoinstallImportHook._install_selenium):
(AutoinstallImportHook):
(AutoinstallImportHook._install_chromedriver):
(AutoinstallImportHook._install_geckodriver):
(AutoinstallImportHook.get_latest_pypi_url):
(AutoinstallImportHook.install_binary):
(autoinstall_everything):
(get_driver_filename):
(get_os_info):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py (modified) (2 diffs)
• Scripts/webkitpy/benchmark_runner/browser_driver/linux_chrome_driver.py (modified) (1 diff)
• Scripts/webkitpy/benchmark_runner/browser_driver/linux_firefox_driver.py (modified) (1 diff)
• Scripts/webkitpy/benchmark_runner/browser_driver/osx_chrome_driver.py (modified) (3 diffs)
• Scripts/webkitpy/benchmark_runner/browser_driver/osx_firefox_driver.py (modified) (3 diffs)
• Scripts/webkitpy/benchmark_runner/browser_driver/osx_safari_driver.py (modified) (1 diff)
• Scripts/webkitpy/benchmark_runner/utils.py (modified) (2 diffs)
• Scripts/webkitpy/common/system/autoinstall.py (modified) (1 diff)
• Scripts/webkitpy/thirdparty/__init__.py (modified) (6 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2017-07-24  Matthew Stewart  <matthew_r_stewart@apple.com>
+
+        Add functions to autoinstaller needed for Benchmark Runner script
+        https://bugs.webkit.org/show_bug.cgi?id=174331
+
+        Reviewed by Dean Johnson, Dewei Zhu, Stephanie Lewis.
+
+        Adds autoinstaller functions to install selenium and webdriver binaries
+        which will be used by the browser driver part of Benchmark Runner.
+
+        * Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py:
+        (BrowserDriver.restore_env):
+        (BrowserDriver):
+        (BrowserDriver.get_webdriver_binary_path):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/linux_chrome_driver.py:
+        (LinuxChromeDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/linux_firefox_driver.py:
+        (LinuxFirefoxDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/osx_chrome_driver.py:
+        (OSXChromeDriver.launch_driver):
+        (OSXChromeCanaryDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/osx_firefox_driver.py:
+        (OSXFirefoxDriver.launch_driver):
+        (OSXFirefoxNightlyDriver.launch_driver):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/osx_safari_driver.py:
+        * Scripts/webkitpy/benchmark_runner/utils.py:
+        (get_driver_binary_path):
+        * Scripts/webkitpy/common/system/autoinstall.py:
+        (AutoInstaller._unzip):
+        * Scripts/webkitpy/thirdparty/__init__.py:
+        (AutoinstallImportHook.find_module):
+        (AutoinstallImportHook._install_selenium):
+        (AutoinstallImportHook):
+        (AutoinstallImportHook._install_chromedriver):
+        (AutoinstallImportHook._install_geckodriver):
+        (AutoinstallImportHook.get_latest_pypi_url):
+        (AutoinstallImportHook.install_binary):
+        (autoinstall_everything):
+        (get_driver_filename):
+        (get_os_info):
+
 2017-07-24  Matthew Stewart  <matthew_r_stewart@apple.com>
 

trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py

@@ -2 +2 @@
 
 from abc import ABCMeta, abstractmethod
+from webkitpy.benchmark_runner.utils import get_driver_binary_path
 
 
@@ -33 +34 @@
     def restore_env(self):
         pass
+
+    @property
+    def webdriver_binary_path(self):
+        return get_driver_binary_path(self.browser_name)

trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/linux_chrome_driver.py

@@ -50 +50 @@
             binary_path = os.path.join(browser_build_path, 'chromium-browser')
             options.binary_location = binary_path
-        driver = webdriver.Chrome(chrome_options=options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Chrome(chrome_options=options, executable_path=driver_executable)
         super(LinuxChromeDriver, self).launch_webdriver(url, driver)
         return driver

trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/linux_firefox_driver.py

@@ -48 +48 @@
             binary_path = os.path.join(browser_build_path, 'firefox-bin')
             options.binary_location = binary_path
-        driver = webdriver.Firefox(firefox_options=options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Firefox(firefox_options=options, executable_path=driver_executable)
         super(LinuxFirefoxDriver, self).launch_webdriver(url, driver)
         return driver

trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_chrome_driver.py

@@ -3 +3 @@
 import logging
 import os
-import subprocess
-import time
 
 from osx_browser_driver import OSXBrowserDriver
@@ -35 +33 @@
             binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
             chrome_options.binary_location = binary_path
-        driver = webdriver.Chrome(chrome_options=chrome_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver
@@ -55 +54 @@
         binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
         chrome_options.binary_location = binary_path
-        driver = webdriver.Chrome(chrome_options=chrome_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Chrome(chrome_options=chrome_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver

trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_firefox_driver.py

@@ -3 +3 @@
 import logging
 import os
-import subprocess
-import time
 
 from osx_browser_driver import OSXBrowserDriver
@@ -31 +29 @@
             binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
             firefox_options.binary_location = binary_path
-        driver = webdriver.Firefox(firefox_options=firefox_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Firefox(firefox_options=firefox_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver
@@ -51 +50 @@
         binary_path = os.path.join(app_path, "Contents/MacOS", self.process_name)
         firefox_options.binary_location = binary_path
-        driver = webdriver.Firefox(firefox_options=firefox_options)
+        driver_executable = self.webdriver_binary_path
+        driver = webdriver.Firefox(firefox_options=firefox_options, executable_path=driver_executable)
         self._launch_webdriver(url=url, driver=driver)
         return driver

trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/osx_safari_driver.py

@@ -3 +3 @@
 import logging
 import os
-import re
 import subprocess
 import time

trunk/Tools/Scripts/webkitpy/benchmark_runner/utils.py

@@ -8 +8 @@
 import shutil
 import sys
+
+from webkitpy.common.memoized import memoized
 
 
@@ -45 +47 @@
         _log.info("Error removing %s: %s" % (path, error))
         pass
+
+
+@memoized
+def get_driver_binary_path(browser_name):
+    if browser_name.startswith('chrome'):
+        import webkitpy.thirdparty.autoinstalled.chromedriver
+        driver_init_file = webkitpy.thirdparty.autoinstalled.chromedriver.__file__
+        driver_executable = os.path.join(os.path.dirname(os.path.realpath(driver_init_file)), 'chromedriver')
+        return driver_executable
+    elif browser_name.startswith('firefox'):
+        import webkitpy.thirdparty.autoinstalled.geckodriver
+        driver_init_file = webkitpy.thirdparty.autoinstalled.geckodriver.__file__
+        driver_executable = os.path.join(os.path.dirname(os.path.realpath(driver_init_file)), 'geckodriver')
+        return driver_executable
 
 

trunk/Tools/Scripts/webkitpy/common/system/autoinstall.py

@@ -276 +276 @@
 
         try:
-            self._extract_all(zip_file, scratch_dir)
+            self._extract_all(zip_file, target_path)
         finally:
             zip_file.close()

trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py

@@ -27 +27 @@
 
 import codecs
+import json
 import os
+import re
 import sys
-
+import urllib2
+
+from collections import namedtuple
+from distutils import spawn
 from webkitpy.common.system.autoinstall import AutoInstaller
 from webkitpy.common.system.filesystem import FileSystem
@@ -35 +40 @@
 _THIRDPARTY_DIR = os.path.dirname(__file__)
 _AUTOINSTALLED_DIR = os.path.join(_THIRDPARTY_DIR, "autoinstalled")
+
+CHROME_DRIVER_URL = "http://chromedriver.storage.googleapis.com/"
+FIREFOX_RELEASES_URL = "https://api.github.com/repos/mozilla/geckodriver/releases"
 
 # Putting the autoinstall code into webkitpy/thirdparty/__init__.py
@@ -47 +55 @@
 #
 #     webkitpy/thirdparty/autoinstalled
+
 fs = FileSystem()
 fs.maybe_make_directory(_AUTOINSTALLED_DIR)
@@ -92 +101 @@
         elif '.twisted_15_5_0' in fullname:
             self._install_twisted_15_5_0()
+        elif '.selenium' in fullname:
+            self._install_selenium()
+        elif '.chromedriver' in fullname:
+            self._install_chromedriver()
+        elif '.geckodriver' in fullname:
+            self._install_geckodriver()
 
     def _install_mechanize(self):
@@ -152 +167 @@
         installer.install(url="https://pypi.python.org/packages/source/z/zope.interface/zope.interface-4.1.3.tar.gz#md5=9ae3d24c0c7415deb249dd1a132f0f79", url_subpath="zope.interface-4.1.3/src/zope")
 
+    def _install_selenium(self):
+        self._ensure_autoinstalled_dir_is_in_sys_path()
+        url, url_subpath = self.get_latest_pypi_url('selenium')
+        self._install(url=url, url_subpath=url_subpath)
+
+    def _install_chromedriver(self):
+        filename_postfix = get_driver_filename().chrome
+        if filename_postfix != "unsupported":
+            version = urllib2.urlopen(CHROME_DRIVER_URL + 'LATEST_RELEASE').read().strip()
+            full_chrome_url = "{base_url}{version}/chromedriver_{os}.zip".format(base_url=CHROME_DRIVER_URL, version=version, os=filename_postfix)
+            self.install_binary(full_chrome_url, 'chromedriver')
+
+    def _install_geckodriver(self):
+        filename_postfix = get_driver_filename().firefox
+        if filename_postfix != "unsupported":
+            firefox_releases_blob = urllib2.urlopen(FIREFOX_RELEASES_URL)
+            firefox_releases_line_separated = json.dumps(json.load(firefox_releases_blob), indent=0).strip()
+            all_firefox_release_urls = "\n".join(re.findall(r'.*browser_download_url.*', firefox_releases_line_separated))
+            full_firefox_url = re.findall(r'.*%s.*' % filename_postfix, all_firefox_release_urls)[0].split('"')[3]
+            self.install_binary(full_firefox_url, 'geckodriver')
+
     def _install(self, url, url_subpath=None, target_name=None):
         installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
         installer.install(url=url, url_subpath=url_subpath, target_name=target_name)
+
+    def get_latest_pypi_url(self, package_name, url_subpath_format='{name}-{version}/{lname}'):
+        json_url = "https://pypi.python.org/pypi/%s/json" % package_name
+        response = urllib2.urlopen(json_url)
+        data = json.load(response)
+        url = data['urls'][1]['url']
+        subpath = url_subpath_format.format(name=package_name, version=data['info']['version'], lname=package_name.lower())
+        return (url, subpath)
+
+    def install_binary(self, url, name):
+        self._install(url=url, target_name=name)
+        directory = os.path.join(_AUTOINSTALLED_DIR, name)
+        os.chmod(os.path.join(directory, name), 0755)
+        open(os.path.join(directory, '__init__.py'), 'w+').close()
 
 
@@ -165 +215 @@
     for method in install_methods:
         getattr(_hook, method)()
+
+def get_driver_filename():
+    os_name, os_type = get_os_info()
+    chrome_os, filefox_os = 'unsupported', 'unsupported'
+    if os_name == 'Linux' and os_type == '64':
+        chrome_os, firefox_os = 'linux64', 'linux64'
+    elif os_name == 'Linux':
+        chrome_os, firefox_os = 'linux32', 'linux32'
+    elif os_name == 'Darwin':
+        chrome_os, firefox_os = 'mac64', 'macos'
+    DriverFilenameForBrowser = namedtuple('DriverFilenameForBrowser', ['chrome', 'firefox'])
+    return DriverFilenameForBrowser(chrome_os, firefox_os)
+
+def get_os_info():
+    import platform
+    os_name = platform.system()
+    os_type = platform.machine()[-2:]
+    return (os_name, os_type)