Changeset 258611 in webkit

Timestamp:

Mar 17, 2020 5:01:42 PM (4 years ago)

Author:

aakash_jain@apple.com

Message:

[ews] Resubmit patches to commit-queue which were cq- by commit-queue and later cq+
​https://bugs.webkit.org/show_bug.cgi?id=208920

Reviewed by Jonathan Bedard.

• BuildSlaveSupport/ews-app/ews/common/buildbot.py:

(Buildbot.update_builder_name_to_id_mapping): Method to generate builder name to id mapping.
(Buildbot.fetch_pending_and_inprogress_builds): Method to fetch pendign and in-progress builds from buildbot.
(Buildbot.get_patches_in_queue): Method to fetch list of patches which are pending or are in-progress on buildbot.

• BuildSlaveSupport/ews-app/ews/common/bugzilla.py:

(Bugzilla.get_cq_plus_timestamp): Get UTC timestamp when cq+ flag was set.

• BuildSlaveSupport/ews-app/ews/fetcher.py:

(FetchLoop.run): Update builder name to id mapping. We just need to update it one-time, since we need the id only
for commit-queue which isn't expected to change.
(BugzillaPatchFetcher.fetch_commit_queue_patches):
(BugzillaPatchFetcher.send_patches_to_buildbot): Allow sending the patch again to commit-queue.
(BugzillaPatchFetcher.patches_to_send_to_commit_queue): Find patches which needs to be sent to commit-queue. Filter
out patches which are already waiting or in-progress on commit-queue, or whose recent build was completed after
setting cq+ flag on bugzilla.

Location:

trunk/Tools

Files:

• BuildSlaveSupport/ews-app/ews/common/bugzilla.py (modified) (2 diffs)
• BuildSlaveSupport/ews-app/ews/common/buildbot.py (modified) (2 diffs)
• BuildSlaveSupport/ews-app/ews/fetcher.py (modified) (6 diffs)
• ChangeLog (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/ews-app/ews/common/bugzilla.py

@@ -34 +34 @@
 import ews.common.util as util
 import ews.config as config
+import dateutil.parser
 
 _log = logging.getLogger(__name__)
@@ -50 +51 @@
         attachment_json['path'] = Bugzilla.file_path_for_patch(attachment_id)
         return attachment_json
+
+    @classmethod
+    def get_cq_plus_timestamp(cls, attachment_id):
+        attachment_json = Bugzilla._fetch_attachment_json(attachment_id)
+        if not attachment_json:
+            _log.warn('Unable to fetch attachment {}.'.format(attachment_id))
+            return None
+
+        for flag in attachment_json.get('flags'):
+            if flag.get('name') == 'commit-queue' and flag.get('status') == '+':
+                try:
+                    return dateutil.parser.parse(flag.get('modification_date'))
+                except:
+                    _log.error('Unable to parse timestamp: {}'.format(flag.get('modification_date')))
+        return None
 
     @classmethod

trunk/Tools/BuildSlaveSupport/ews-app/ews/common/buildbot.py

@@ -40 +40 @@
     SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, CANCELLED = ALL_RESULTS
     icons_for_queues_mapping = {}
+    builder_name_to_id_mapping = {}
 
     @classmethod
@@ -100 +101 @@
             Buildbot.icons_for_queues_mapping[shortname] = builder.get('icon')
 
-        return Buildbot.icons_for_queues_mapping
+    @classmethod
+    def update_builder_name_to_id_mapping(cls):
+        url = 'https://{}/api/v2/builders'.format(config.BUILDBOT_SERVER_HOST)
+        builders_data = util.fetch_data_from_url(url)
+        if not builders_data:
+            return
+        for builder in builders_data.json().get('builders', []):
+            name = builder.get('name')
+            Buildbot.builder_name_to_id_mapping[name] = builder.get('builderid')
+
+    @classmethod
+    def fetch_pending_and_inprogress_builds(cls, builder_full_name):
+        builderid = Buildbot.builder_name_to_id_mapping.get(builder_full_name)
+        if not builderid:
+            _log.error('Invalid builder: {}'.format(builder_full_name))
+            return {}
+        url = 'https://{}/api/v2/builders/{}/buildrequests?complete=false&property=*'.format(config.BUILDBOT_SERVER_HOST, builderid)
+        builders_data = util.fetch_data_from_url(url)
+        if not builders_data:
+            return {}
+        return builders_data.json()
+
+    @classmethod
+    def get_patches_in_queue(cls, builder_full_name):
+        patch_ids = []
+        builds = cls.fetch_pending_and_inprogress_builds(builder_full_name)
+        for buildrequest in builds.get('buildrequests', []):
+            properties = buildrequest.get('properties')
+            if properties:
+                patch_ids.append(properties.get('patch_id')[0])
+        _log.debug('Patches in queue for {}: {}'.format(builder_full_name, patch_ids))
+        return patch_ids
 
     @classmethod

trunk/Tools/BuildSlaveSupport/ews-app/ews/fetcher.py

@@ -21 +21 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import datetime
 import logging
+import pytz
 import threading
 import time
@@ -28 +30 @@
 from ews.common.buildbot import Buildbot
 from ews.models.patch import Patch
+from ews.views.statusbubble import StatusBubble
 
 _log = logging.getLogger(__name__)
@@ -40 +43 @@
 
     def run(self):
+        Buildbot.update_builder_name_to_id_mapping()
         while True:
             Buildbot.update_icons_for_queues_mapping()
@@ -70 +74 @@
         _log.debug('cq+ patches: {}'.format(patch_ids_commit_queue))
         Patch.save_patches(patch_ids_commit_queue)
-        patches_to_send = self.patches_to_send_to_buildbot(patch_ids_commit_queue, commit_queue=True)
+        patches_to_send = self.patches_to_send_to_commit_queue(patch_ids_commit_queue)
         _log.info('{} cq+ patches, {} patches need to be sent to commit queue: {}'.format(len(patch_ids_commit_queue), len(patches_to_send), patches_to_send))
         self.send_patches_to_buildbot(patches_to_send, send_to_commit_queue=True)
@@ -86 +90 @@
                 Patch.set_obsolete(patch_id)
                 continue
-            if Patch.is_patch_sent_to_buildbot(patch_id, commit_queue=send_to_commit_queue):
-                _log.error('Patch {} is already sent to buildbot/commit-queue.'.format(patch_id))
+            if not send_to_commit_queue and Patch.is_patch_sent_to_buildbot(patch_id):
+                _log.error('Patch {} is already sent to buildbot.'.format(patch_id))
                 continue
             Patch.set_sent_to_buildbot(patch_id, True, commit_queue=send_to_commit_queue)
@@ -100 +104 @@
                 #FIXME: send an email for this failure
 
-    def patches_to_send_to_buildbot(self, patch_ids, commit_queue=False):
-        return [patch_id for patch_id in patch_ids if not Patch.is_patch_sent_to_buildbot(patch_id, commit_queue)]
+    def patches_to_send_to_buildbot(self, patch_ids):
+        return [patch_id for patch_id in patch_ids if not Patch.is_patch_sent_to_buildbot(patch_id)]
+
+    def patches_to_send_to_commit_queue(self, patch_ids):
+        if not patch_ids:
+            return patch_ids
+        patches_in_queue = set(Buildbot.get_patches_in_queue('Commit-Queue'))
+        patch_ids = [patch_id for patch_id in set(patch_ids) if str(patch_id) not in patches_in_queue]
+
+        patch_ids_to_send = []
+        for patch_id in patch_ids:
+            patch = Patch.get_patch(patch_id)
+            recent_build, _ = StatusBubble().get_latest_build_for_queue(patch, 'commit')
+            if not recent_build:
+                patch_ids_to_send.append(patch_id)
+                continue
+            recent_build_timestamp = datetime.datetime.fromtimestamp(recent_build.complete_at, tz=pytz.UTC)
+            cq_timestamp = Bugzilla.get_cq_plus_timestamp(patch_id)
+            if not cq_timestamp:
+                patch_ids_to_send.append(patch_id)
+                continue
+            if cq_timestamp > recent_build_timestamp:
+                patch_ids_to_send.append(patch_id)
+        return patch_ids_to_send
 
     @classmethod

trunk/Tools/ChangeLog

@@ +1 @@
+2020-03-17  Aakash Jain  <aakash_jain@apple.com>
+
+        [ews] Resubmit patches to commit-queue which were cq- by commit-queue and later cq+
+        https://bugs.webkit.org/show_bug.cgi?id=208920
+
+        Reviewed by Jonathan Bedard.
+
+        * BuildSlaveSupport/ews-app/ews/common/buildbot.py:
+        (Buildbot.update_builder_name_to_id_mapping): Method to generate builder name to id mapping.
+        (Buildbot.fetch_pending_and_inprogress_builds): Method to fetch pendign and in-progress builds from buildbot.
+        (Buildbot.get_patches_in_queue): Method to fetch list of patches which are pending or are in-progress on buildbot.
+        * BuildSlaveSupport/ews-app/ews/common/bugzilla.py:
+        (Bugzilla.get_cq_plus_timestamp): Get UTC timestamp when cq+ flag was set.
+        * BuildSlaveSupport/ews-app/ews/fetcher.py:
+        (FetchLoop.run): Update builder name to id mapping. We just need to update it one-time, since we need the id only
+        for commit-queue which isn't expected to change.
+        (BugzillaPatchFetcher.fetch_commit_queue_patches):
+        (BugzillaPatchFetcher.send_patches_to_buildbot): Allow sending the patch again to commit-queue.
+        (BugzillaPatchFetcher.patches_to_send_to_commit_queue): Find patches which needs to be sent to commit-queue. Filter
+        out patches which are already waiting or in-progress on commit-queue, or whose recent build was completed after
+        setting cq+ flag on bugzilla.
+
 2020-03-17  Chris Dumez  <cdumez@apple.com>