Changeset 85080 in webkit


Ignore:
Timestamp:
Apr 27, 2011 1:44:15 PM (13 years ago)
Author:
tony@chromium.org
Message:

2011-04-27 Tony Chang <tony@chromium.org>

Reviewed by Ojan Vafai.

lazily autoinstall thirdparty python libraries
https://bugs.webkit.org/show_bug.cgi?id=55723

This helps by only having pywebsocket installed when the bots run. It
doesn't help much for devs since webkit-patch pulls in almost everything.

  • Scripts/webkitpy/common/net/bugzilla/bugzilla.py:
  • Scripts/webkitpy/common/net/buildbot/buildbot.py:
  • Scripts/webkitpy/common/net/networktransaction.py:
  • Scripts/webkitpy/common/net/networktransaction_unittest.py:
  • Scripts/webkitpy/common/net/statusserver.py:
  • Scripts/webkitpy/layout_tests/port/websocket_server.py:
  • Scripts/webkitpy/thirdparty/init.py: Add an import hook so we can install based on

the import command rather than just installing everything.

  • Scripts/webkitpy/thirdparty/init_unittest.py: Added.
Location:
trunk/Tools
Files:
1 added
8 edited

Legend:

Unmodified
Added
Removed
  • trunk/Tools/ChangeLog

    r85076 r85080  
     12011-04-27  Tony Chang  <tony@chromium.org>
     2
     3        Reviewed by Ojan Vafai.
     4
     5        lazily autoinstall thirdparty python libraries
     6        https://bugs.webkit.org/show_bug.cgi?id=55723
     7
     8        This helps by only having pywebsocket installed when the bots run.  It
     9        doesn't help much for devs since webkit-patch pulls in almost everything.
     10
     11        * Scripts/webkitpy/common/net/bugzilla/bugzilla.py:
     12        * Scripts/webkitpy/common/net/buildbot/buildbot.py:
     13        * Scripts/webkitpy/common/net/networktransaction.py:
     14        * Scripts/webkitpy/common/net/networktransaction_unittest.py:
     15        * Scripts/webkitpy/common/net/statusserver.py:
     16        * Scripts/webkitpy/layout_tests/port/websocket_server.py:
     17        * Scripts/webkitpy/thirdparty/__init__.py: Add an import hook so we can install based on
     18            the import command rather than just installing everything.
     19        * Scripts/webkitpy/thirdparty/__init___unittest.py: Added.
     20
    1212011-04-27  Girish Ramakrishnan  <girish@forwardbias.in>
    222
  • trunk/Tools/Scripts/webkitpy/common/net/bugzilla/bugzilla.py

    r84575 r85080  
    4646from webkitpy.common.net.credentials import Credentials
    4747from webkitpy.common.system.user import User
    48 from webkitpy.thirdparty.autoinstalled.mechanize import Browser
    4948from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, SoupStrainer
    5049
     
    216215        # FIXME: We should use some sort of Browser mock object when in dryrun
    217216        # mode (to prevent any mistakes).
     217        from webkitpy.thirdparty.autoinstalled.mechanize import Browser
    218218        self.browser = Browser()
    219219        # Ignore bugs.webkit.org/robots.txt until we fix it to allow this
  • trunk/Tools/Scripts/webkitpy/common/net/buildbot/buildbot.py

    r84676 r85080  
    4747from webkitpy.common.system.zipfileset import ZipFileSet
    4848from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
    49 from webkitpy.thirdparty.autoinstalled.mechanize import Browser
    5049
    5150_log = get_logger(__file__)
     
    5857        self._builds_cache = {}
    5958        self._revision_to_build_number = None
     59        from webkitpy.thirdparty.autoinstalled.mechanize import Browser
    6060        self._browser = Browser()
    6161        self._browser.set_handle_robots(False) # The builder pages are excluded by robots.txt
  • trunk/Tools/Scripts/webkitpy/common/net/networktransaction.py

    r75230 r85080  
    3030import time
    3131
    32 from webkitpy.thirdparty.autoinstalled import mechanize
    3332from webkitpy.common.system.deprecated_logging import log
    3433
     
    5150        self._total_sleep = 0
    5251        self._backoff_seconds = self._initial_backoff_seconds
     52        from webkitpy.thirdparty.autoinstalled import mechanize
    5353        while True:
    5454            try:
  • trunk/Tools/Scripts/webkitpy/common/net/networktransaction_unittest.py

    r75230 r85080  
    3131from webkitpy.common.net.networktransaction import NetworkTransaction, NetworkTimeout
    3232from webkitpy.common.system.logtesting import LoggingTestCase
    33 from webkitpy.thirdparty.autoinstalled.mechanize import HTTPError
    3433
    3534
     
    6059        self._run_count += 1
    6160        if self._run_count < 3:
     61            from webkitpy.thirdparty.autoinstalled.mechanize import HTTPError
    6262            raise HTTPError("http://example.com/", 500, "internal server error", None, None)
    6363        return 42
    6464
    6565    def _raise_404_error(self):
     66        from webkitpy.thirdparty.autoinstalled.mechanize import HTTPError
    6667        raise HTTPError("http://foo.com/", 404, "not found", None, None)
    6768
  • trunk/Tools/Scripts/webkitpy/common/net/statusserver.py

    r83614 r85080  
    3131from webkitpy.common.net.networktransaction import NetworkTransaction
    3232from webkitpy.common.system.deprecated_logging import log
    33 from webkitpy.thirdparty.autoinstalled.mechanize import Browser
    3433from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
    3534
     
    4746    def __init__(self, host=default_host, browser=None, bot_id=None):
    4847        self.set_host(host)
     48        from webkitpy.thirdparty.autoinstalled.mechanize import Browser
    4949        self._browser = browser or Browser()
    5050        self.set_bot_id(bot_id)
  • trunk/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py

    r76547 r85080  
    4747
    4848from webkitpy.common.system.executive import Executive
    49 from webkitpy.thirdparty.autoinstalled.pywebsocket import mod_pywebsocket
    5049
    5150
     
    158157        self._wsout = codecs.open(output_log, "w", "utf-8")
    159158
     159        from webkitpy.thirdparty.autoinstalled.pywebsocket import mod_pywebsocket
    160160        python_interp = sys.executable
    161161        pywebsocket_base = os.path.join(
  • trunk/Tools/Scripts/webkitpy/thirdparty/__init__.py

    r81348 r85080  
    2929import codecs
    3030import os
     31import sys
    3132
    3233from webkitpy.common.system.autoinstall import AutoInstaller
     34from webkitpy.common.system.filesystem import FileSystem
     35
     36_THIRDPARTY_DIR = os.path.dirname(__file__)
     37_AUTOINSTALLED_DIR = os.path.join(_THIRDPARTY_DIR, "autoinstalled")
    3338
    3439# Putting the autoinstall code into webkitpy/thirdparty/__init__.py
     
    3641# webkitpy.thirdparty.  This is useful if the caller wants to configure
    3742# logging prior to executing autoinstall code.
    38 
    39 # FIXME: Ideally, a package should be autoinstalled only if the caller
    40 #        attempts to import from that individual package.  This would
    41 #        make autoinstalling lazier than it is currently.  This can
    42 #        perhaps be done using Python's import hooks as the original
    43 #        autoinstall implementation did.
    4443
    4544# FIXME: If any of these servers is offline, webkit-patch breaks (and maybe
     
    4948#
    5049#     webkitpy/thirdparty/autoinstalled
    51 thirdparty_dir = os.path.dirname(__file__)
    52 autoinstalled_dir = os.path.join(thirdparty_dir, "autoinstalled")
     50fs = FileSystem()
     51fs.maybe_make_directory(_AUTOINSTALLED_DIR)
    5352
    54 # We need to download ClientForm since the mechanize package that we download
    55 # below requires it.  The mechanize package uses ClientForm, for example,
    56 # in _html.py.  Since mechanize imports ClientForm in the following way,
    57 #
    58 # > import sgmllib, ClientForm
    59 #
    60 # the search path needs to include ClientForm.  We put ClientForm in
    61 # its own directory so that we can include it in the search path without
    62 # including other modules as a side effect.
    63 clientform_dir = os.path.join(autoinstalled_dir, "clientform")
    64 installer = AutoInstaller(append_to_search_path=True,
    65                           target_dir=clientform_dir)
    66 installer.install(url="http://pypi.python.org/packages/source/C/ClientForm/ClientForm-0.2.10.zip",
    67                   url_subpath="ClientForm.py")
     53init_path = fs.join(_AUTOINSTALLED_DIR, "__init__.py")
     54if not fs.exists(init_path):
     55    fs.write_text_file(init_path, "")
    6856
    69 # The remaining packages do not need to be in the search path, so we create
    70 # a new AutoInstaller instance that does not append to the search path.
    71 installer = AutoInstaller(target_dir=autoinstalled_dir)
     57readme_path = fs.join(_AUTOINSTALLED_DIR, "README")
     58if not fs.exists(readme_path):
     59    fs.write_text_file(readme_path,
     60        "This directory is auto-generated by WebKit and is "
     61        "safe to delete.\nIt contains needed third-party Python "
     62        "packages automatically downloaded from the web.")
    7263
    73 installer.install(url="http://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.4.zip",
    74                   url_subpath="mechanize")
    75 installer.install(url="http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b",
    76                   url_subpath="pep8-0.5.0/pep8.py")
    77 installer.install(url="http://www.adambarth.com/webkit/eliza",
    78                   target_name="eliza.py")
    7964
    80 # Since irclib and ircbot are two top-level packages, we need to import
    81 # them separately.  We group them into an irc package for better
    82 # organization purposes.
    83 irc_dir = os.path.join(autoinstalled_dir, "irc")
    84 installer = AutoInstaller(target_dir=irc_dir)
    85 installer.install(url="http://downloads.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip", url_subpath="irclib.py")
    86 installer.install(url="http://downloads.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip", url_subpath="ircbot.py")
     65class AutoinstallImportHook(object):
     66    def __init__(self, filesystem=None):
     67        self._fs = filesystem or FileSystem()
    8768
    88 pywebsocket_dir = os.path.join(autoinstalled_dir, "pywebsocket")
    89 installer = AutoInstaller(target_dir=pywebsocket_dir)
    90 installer.install(url="http://pywebsocket.googlecode.com/files/mod_pywebsocket-0.5.2.tar.gz",
    91                   url_subpath="pywebsocket-0.5.2/src/mod_pywebsocket")
     69    def find_module(self, fullname, path):
     70        # This method will run before each import. See http://www.python.org/dev/peps/pep-0302/
     71        if '.autoinstalled' not in fullname:
     72            return
    9273
    93 readme_path = os.path.join(autoinstalled_dir, "README")
    94 if not os.path.exists(readme_path):
    95     with codecs.open(readme_path, "w", "ascii") as file:
    96         file.write("This directory is auto-generated by WebKit and is "
    97                    "safe to delete.\nIt contains needed third-party Python "
    98                    "packages automatically downloaded from the web.")
     74        if '.mechanize' in fullname:
     75            self._install_mechanize()
     76        elif '.pep8' in fullname:
     77            self._install_pep8()
     78        elif '.eliza' in fullname:
     79            self._install_eliza()
     80        elif '.irc' in fullname:
     81            self._install_irc()
     82        elif '.pywebsocket' in fullname:
     83            self._install_pywebsocket()
     84
     85    def _install_mechanize(self):
     86        # The mechanize package uses ClientForm, for example, in _html.py.
     87        # Since mechanize imports ClientForm in the following way,
     88        #
     89        # > import sgmllib, ClientForm
     90        #
     91        # the search path needs to include ClientForm.  We put ClientForm in
     92        # its own directory so that we can include it in the search path
     93        # without including other modules as a side effect.
     94        clientform_dir = self._fs.join(_AUTOINSTALLED_DIR, "clientform")
     95        installer = AutoInstaller(append_to_search_path=True,
     96                                  target_dir=clientform_dir)
     97        installer.install(url="http://pypi.python.org/packages/source/C/ClientForm/ClientForm-0.2.10.zip",
     98                          url_subpath="ClientForm.py")
     99
     100        self._install("http://pypi.python.org/packages/source/m/mechanize/mechanize-0.2.4.zip",
     101                      "mechanize")
     102
     103    def _install_pep8(self):
     104        self._install("http://pypi.python.org/packages/source/p/pep8/pep8-0.5.0.tar.gz#md5=512a818af9979290cd619cce8e9c2e2b",
     105                      "pep8-0.5.0/pep8.py")
     106
     107    def _install_eliza(self):
     108        installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
     109        installer.install(url="http://www.adambarth.com/webkit/eliza",
     110                          target_name="eliza.py")
     111
     112    def _install_irc(self):
     113        # Since irclib and ircbot are two top-level packages, we need to import
     114        # them separately.  We group them into an irc package for better
     115        # organization purposes.
     116        irc_dir = self._fs.join(_AUTOINSTALLED_DIR, "irc")
     117        installer = AutoInstaller(target_dir=irc_dir)
     118        installer.install(url="http://downloads.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip",
     119                          url_subpath="irclib.py")
     120        installer.install(url="http://downloads.sourceforge.net/project/python-irclib/python-irclib/0.4.8/python-irclib-0.4.8.zip",
     121                          url_subpath="ircbot.py")
     122
     123    def _install_pywebsocket(self):
     124        pywebsocket_dir = self._fs.join(_AUTOINSTALLED_DIR, "pywebsocket")
     125        installer = AutoInstaller(target_dir=pywebsocket_dir)
     126        installer.install(url="http://pywebsocket.googlecode.com/files/mod_pywebsocket-0.5.2.tar.gz",
     127                          url_subpath="pywebsocket-0.5.2/src/mod_pywebsocket")
     128
     129    def _install(self, url, url_subpath):
     130        installer = AutoInstaller(target_dir=_AUTOINSTALLED_DIR)
     131        installer.install(url=url, url_subpath=url_subpath)
     132
     133
     134sys.meta_path.append(AutoinstallImportHook())
Note: See TracChangeset for help on using the changeset viewer.