Changeset 287045 in webkit

Timestamp:

Dec 14, 2021, 1:31:59 PM (5 years ago)

Author:

Jonathan Bedard

Message:

[reporelaypy] Update checkout with hook instead of polling
​https://bugs.webkit.org/show_bug.cgi?id=234243
<rdar://problem/86413065>

Reviewed by Dewei Zhu.

• Tools/Scripts/libraries/reporelaypy/reporelaypy/init.py: Bump version,

export HookProcessor and HookReceiver.

• Tools/Scripts/libraries/reporelaypy/reporelaypy/checkout.py:

(Checkout.update_for): If a branch is new, we need to track it.

• Tools/Scripts/libraries/reporelaypy/reporelaypy/database.py:

(Database.init): Use self.host and self.password.

• Tools/Scripts/libraries/reporelaypy/reporelaypy/hooks.py: Added.

(HookProcessor): Class to process received hooks.
(HookReceiver): Class to receive hooks and queue them for processing.

• Tools/Scripts/libraries/reporelaypy/reporelaypy/tests/hooks_unittest.py: Added.

(HooksUnittest):
(HooksUnittest.setUp):
(HooksUnittest.test_receive):
(HooksUnittest.test_invalid):
(HooksUnittest.test_process):
(HooksUnittest.test_hmac):
(HooksUnittest.test_invalid_hmac):

• Tools/Scripts/libraries/reporelaypy/reporelaypy/webserver.py: Add hook routes,

if hooks are enabled.

• Tools/Scripts/libraries/reporelaypy/run: Allow caller to enable hooks.
• Tools/Scripts/libraries/reporelaypy/setup.py: Bump version.

Canonical link: ​https://commits.webkit.org/245242@main

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/libraries/reporelaypy/reporelaypy/__init__.py (modified) (2 diffs)
• Scripts/libraries/reporelaypy/reporelaypy/checkout.py (modified) (2 diffs)
• Scripts/libraries/reporelaypy/reporelaypy/database.py (modified) (1 diff)
• Scripts/libraries/reporelaypy/reporelaypy/hooks.py (added)
• Scripts/libraries/reporelaypy/reporelaypy/tests/hooks_unittest.py (added)
• Scripts/libraries/reporelaypy/reporelaypy/webserver.py (modified) (4 diffs)
• Scripts/libraries/reporelaypy/run (modified) (4 diffs)
• Scripts/libraries/reporelaypy/setup.py (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2021-12-13  Jonathan Bedard  <jbedard@apple.com>
+
+        [reporelaypy] Update checkout with hook instead of polling
+        https://bugs.webkit.org/show_bug.cgi?id=234243
+        <rdar://problem/86413065>
+
+        Reviewed by Dewei Zhu.
+
+        * Scripts/libraries/reporelaypy/reporelaypy/__init__.py: Bump version,
+        export HookProcessor and HookReceiver.
+        * Scripts/libraries/reporelaypy/reporelaypy/checkout.py:
+        (Checkout.update_for): If a branch is new, we need to track it.
+        * Scripts/libraries/reporelaypy/reporelaypy/database.py:
+        (Database.__init__): Use self.host and self.password.
+        * Scripts/libraries/reporelaypy/reporelaypy/hooks.py: Added.
+        (HookProcessor): Class to process received hooks.
+        (HookReceiver): Class to receive hooks and queue them for processing.
+        * Scripts/libraries/reporelaypy/reporelaypy/tests/hooks_unittest.py: Added.
+        (HooksUnittest):
+        (HooksUnittest.setUp):
+        (HooksUnittest.test_receive):
+        (HooksUnittest.test_invalid):
+        (HooksUnittest.test_process):
+        (HooksUnittest.test_hmac):
+        (HooksUnittest.test_invalid_hmac):
+        * Scripts/libraries/reporelaypy/reporelaypy/webserver.py: Add hook routes,
+        if hooks are enabled.
+        * Scripts/libraries/reporelaypy/run: Allow caller to enable hooks.
+        * Scripts/libraries/reporelaypy/setup.py: Bump version.
+
 2021-12-14  Alex Christensen  <achristensen@webkit.org>
 

trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/__init__.py

@@ -45 +45 @@
     )
 
-version = Version(0, 2, 0)
+version = Version(0, 3, 0)
 
 import webkitflaskpy
@@ -52 +52 @@
 from reporelaypy.database import Database
 from reporelaypy.checkoutroute import CheckoutRoute, Redirector
+from reporelaypy.hooks import HookProcessor, HookReceiver
 
 AutoInstall.register(Package('fakeredis', Version(1, 5, 2)))

trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/checkout.py

@@ -165 +165 @@
         return False
 
-    def update_for(self, branch=None, remote='origin'):
+    def update_for(self, branch=None, remote='origin', track=False):
         if not self.repository:
             sys.stderr.write("Cannot update '{}', clone still pending...\n".format(branch))
@@ -171 +171 @@
 
         branch = branch or self.repository.default_branch
-        if branch == self.repository.default_branch:
+        if not self.repository.prod_branches.match(branch):
+            return False
+        elif track and branch not in self.repository.branches_for(remote=remote):
+            run(
+                [self.repository.executable(), 'branch', '--track', branch, 'remotes/{}/{}'.format(remote, branch)],
+                cwd=self.repository.root_path,
+            )
+            self.repository.cache.populate(branch=branch)
+        elif branch == self.repository.default_branch:
             self.repository.pull(remote=remote)
             self.repository.cache.populate(branch=branch)
             return True
-        if not self.repository.prod_branches.match(branch):
-            return False
-        if self.is_updated(branch, remote=remote):
+        elif not track and self.is_updated(branch, remote=remote):
             return True
 

trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/database.py

@@ -47 +47 @@
         self.password = password or Environment.instance().get(self.PASSWORD_ENV)
 
-        if host:
+        if self.host:
             import redis
-            self._redis = redis.Redis(host=host, password=password)
+            self._redis = redis.StrictRedis(host=self.host, password=self.password)
         else:
             import fakeredis

trunk/Tools/Scripts/libraries/reporelaypy/reporelaypy/webserver.py

@@ -21 +21 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import json
 import os
 
@@ -29 +30 @@
 
 from flask import Flask, current_app, json as fjson
-from reporelaypy import Checkout, CheckoutRoute, Database, Redirector
+from reporelaypy import Checkout, CheckoutRoute, Database, Redirector, HookReceiver
 
 app = Flask(__name__)
@@ -40 +41 @@
 )
 
+hook_args = json.loads(os.environ.get('HOOKS', '{}'))
+if hook_args.get('enabled', False):
+    hook_routes = HookReceiver(
+        import_name=__name__, database=database,
+        debug=hook_args.get('debug', False), secret=os.environ.get(HookReceiver.SECRET_ENV),
+    )
+else:
+    hook_routes = None
+
 
 @app.route('/__health')
@@ -47 +57 @@
 
 app.register_blueprint(checkout_routes)
+if hook_routes:
+    app.register_blueprint(hook_routes)
 
 

trunk/Tools/Scripts/libraries/reporelaypy/run

@@ -39 +39 @@
     import webkitpy
 
-from reporelaypy import Checkout, Database, Redirector
+from reporelaypy import Checkout, Database, HookProcessor, HookReceiver, Redirector
 from webkitcorepy import arguments, AutoInstall
 from whichcraft import which
@@ -102 +102 @@
     )
 
+    group = parser.add_argument_group('Hooks')
+    group.add_argument(
+        '--hooks', '--no-hooks', action=arguments.NoAction, dest='hooks', default=False,
+        help='Enable or disable hook end-points (disabled by default)',
+    )
+    group.add_argument(
+        '--debug', '--no-debug', action=arguments.NoAction, dest='hooks_debug', default=True,
+        help='Enable endpoint to report hooks being processed (enabled by default)',
+    )
+
     args = parser.parse_args(args=args)
 
@@ -141 +151 @@
             print('    {}: {}'.format(redirector.name, redirector.url))
 
-    env = dict(
+    passenv = dict(
         PYTHONPATH=':'.join(sys.path),
         CHECKOUT=json.dumps(checkout, cls=Checkout.Encoder),
-        REDIRECTORS=json.dumps([Redirector(url) for url in args.redirector or []], cls=Redirector.Encoder)
+        REDIRECTORS=json.dumps([Redirector(url) for url in args.redirector or []], cls=Redirector.Encoder),
+        HOOKS=json.dumps({'enabled': args.hooks, 'debug': args.hooks_debug}),
     )
 
     if AutoInstall.directory:
-        env['AUTOINSTALL_PATH'] = AutoInstall.directory
+        passenv['AUTOINSTALL_PATH'] = AutoInstall.directory
     if database.host:
-        env[database.HOST_ENV] = database.host
+        passenv[database.HOST_ENV] = database.host
     if database.password:
-        env[database.PASSWORD_ENV] = database.password
+        passenv[database.PASSWORD_ENV] = database.password
     if database.default_expiration:
-        env[database.EXPIRATION_ENV] = str(database.default_expiration)
+        passenv[database.EXPIRATION_ENV] = str(database.default_expiration)
+    if os.environ.get(HookReceiver.SECRET_ENV):
+        passenv[HookReceiver.SECRET_ENV] = os.environ.get(HookReceiver.SECRET_ENV)
+
+    processor = HookProcessor(checkout=checkout, database=database) if args.hooks else None
 
     with subprocess.Popen(
         [which('gunicorn'), 'reporelaypy.webserver:app'],
         cwd=os.path.dirname(os.path.dirname(reporelaypy.__file__)),
-        env=env,
+        env=passenv,
     ) as webserver:
         last_poll = time.time()
@@ -170 +185 @@
                 last_poll = time.time()
             if last_pull + args.update_interval < time.time():
-                checkout.update_all()
+                processor.process_hooks() if processor else checkout.update_all()
                 last_pull = time.time()
             time.sleep(math.gcd(args.poll, args.update_interval))

trunk/Tools/Scripts/libraries/reporelaypy/setup.py

@@ -31 +31 @@
 setup(
     name='reporelaypy',
-    version='0.2.0',
+    version='0.3.0',
     description='Library for visualizing, processing and storing test results.',
     long_description=readme(),