Changeset 225433 in webkit


Ignore:
Timestamp:
Dec 1, 2017, 4:39:32 PM (7 years ago)
Author:
aakash_jain@apple.com
Message:

[build.webkit.org] Move python code to load config from master.cfg in separate file
​https://bugs.webkit.org/show_bug.cgi?id=180278

Reviewed by Daniel Bates.

  • BuildSlaveSupport/build.webkit.org-config/master.cfg: Moved appropriate code to loadConfig.py
  • BuildSlaveSupport/build.webkit.org-config/loadConfig.py: Ditto.
  • BuildSlaveSupport/build.webkit.org-config/steps.py: Removed extra import.
Location:
trunk/Tools
Files:
3 edited
1 copied

Legend:

Unmodified
Added
Removed

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/loadConfig.py

    r225432 r225433  
    1 # -*- python -*-
    2 # ex: set syntax=python:
     1# Copyright (C) 2017 Apple Inc. All rights reserved.
     2#
     3# Redistribution and use in source and binary forms, with or without
     4# modification, are permitted provided that the following conditions
     5# are met:
     6# 1.  Redistributions of source code must retain the above copyright
     7#     notice, this list of conditions and the following disclaimer.
     8# 2.  Redistributions in binary form must reproduce the above copyright
     9#     notice, this list of conditions and the following disclaimer in the
     10#     documentation and/or other materials provided with the distribution.
     11#
     12# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
     13# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
     14# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
     15# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
     16# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
     17# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
     18# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
     19# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
     20# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
     21# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    322
    423from buildbot.buildslave import BuildSlave
    5 from buildbot.changes.pb import PBChangeSource
    624from buildbot.scheduler import AnyBranchScheduler, Triggerable
    725from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
    826from buildbot.schedulers.filter import ChangeFilter
    9 from buildbot.status import html
    10 from buildbot.status.web.authz import Authz
    1127from buildbot.process import buildstep, factory, properties
    1228
    1329from factories import *
    14 from twisted.internet import defer
    1530
    1631import re
    … …  
    2136import wkbuild
    2237
    23 APPLE_WEBKIT_AWS_PROXY = "http://54.190.50.182:873"
    24 S3URL = "https://s3-us-west-2.amazonaws.com/"
    25 
    26 c = BuildmasterConfig = {}
    27 
    28 c['change_source'] = PBChangeSource(port=16000)
    29 
    30 # permissions for WebStatus
    31 authz = Authz(
    32     auth=CommitterAuth('auth.json'),
    33     forceBuild='auth',
    34     forceAllBuilds='auth',
    35     pingBuilder=True,
    36     gracefulShutdown=False,
    37     stopBuild='auth',
    38     stopAllBuilds='auth',
    39     cancelPendingBuild='auth',
    40     stopChange=True,
    41     cleanShutdown=False)
    42 
    43 c['status'] = []
    44 c['status'].append(html.WebStatus(http_port=8710,
    45                                   revlink="https://trac.webkit.org/changeset/%s",
    46                                   changecommentlink=(r"(https://bugs\.webkit\.org/show_bug\.cgi\?id=|webkit\.org/b/)(\d+)", r"https://bugs.webkit.org/show_bug.cgi?id=\2"),
    47                                   authz=authz))
    48 
    49 c['slavePortnum'] = 17000
    50 c['projectName'] = "WebKit"
    51 c['projectURL'] = "https://webkit.org"
    52 c['buildbotURL'] = "https://build.webkit.org/"
    53 
    54 c['buildHorizon'] = 1000
    55 c['logHorizon'] = 500
    56 c['eventHorizon'] = 200
    57 c['buildCacheSize'] = 60
     38trunk_filter = ChangeFilter(branch=["trunk", None])
    5839
    5940
    … …  
    6142    return max(requests, key=operator.attrgetter("submittedAt"))
    6243
    63 
    64 class PlatformSpecificScheduler(AnyBranchScheduler):
    65     def __init__(self, platform, branch, **kwargs):
    66         self.platform = platform
    67         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
    68         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
    69 
    70     def filter(self, change):
    71         return wkbuild.should_build(self.platform, change.files)
    72 
    73 trunk_filter = ChangeFilter(branch=["trunk", None])
    7444
    7545def loadBuilderConfig(c):
    … …  
    150120        c['builders'].append(builder)
    151121
    152 loadBuilderConfig(c)
     122
     123class PlatformSpecificScheduler(AnyBranchScheduler):
     124    def __init__(self, platform, branch, **kwargs):
     125        self.platform = platform
     126        filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
     127        AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
     128
     129    def filter(self, change):
     130        return wkbuild.should_build(self.platform, change.files)

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/master.cfg

    r225334 r225433  
    22# ex: set syntax=python:
    33
    4 from buildbot.buildslave import BuildSlave
    54from buildbot.changes.pb import PBChangeSource
    6 from buildbot.scheduler import AnyBranchScheduler, Triggerable
    7 from buildbot.schedulers.forcesched import FixedParameter, ForceScheduler, StringParameter, BooleanParameter
    8 from buildbot.schedulers.filter import ChangeFilter
    95from buildbot.status import html
    106from buildbot.status.web.authz import Authz
    11 from buildbot.process import buildstep, factory, properties
    12 
    13 from factories import *
    14 from twisted.internet import defer
    15 
    16 import re
    17 import json
    18 import operator
    197
    208from committer_auth import CommitterAuth
    21 import wkbuild
     9from loadConfig import *
    2210
    23 APPLE_WEBKIT_AWS_PROXY = "http://54.190.50.182:873"
    24 S3URL = "https://s3-us-west-2.amazonaws.com/"
    2511
    2612c = BuildmasterConfig = {}
    … …  
    5844
    5945
    60 def pickLatestBuild(builder, requests):
    61     return max(requests, key=operator.attrgetter("submittedAt"))
    62 
    63 
    64 class PlatformSpecificScheduler(AnyBranchScheduler):
    65     def __init__(self, platform, branch, **kwargs):
    66         self.platform = platform
    67         filter = ChangeFilter(branch=[branch, None], filter_fn=self.filter)
    68         AnyBranchScheduler.__init__(self, name=platform, change_filter=filter, **kwargs)
    69 
    70     def filter(self, change):
    71         return wkbuild.should_build(self.platform, change.files)
    72 
    73 trunk_filter = ChangeFilter(branch=["trunk", None])
    74 
    75 def loadBuilderConfig(c):
    76     # FIXME: These file handles are leaked.
    77     passwords = json.load(open('passwords.json'))
    78     config = json.load(open('config.json'))
    79 
    80     c['slaves'] = [BuildSlave(slave['name'], passwords[slave['name']], max_builds=1) for slave in config['slaves']]
    81 
    82     c['schedulers'] = []
    83     for scheduler in config['schedulers']:
    84         if "change_filter" in scheduler:
    85             scheduler["change_filter"] = globals()[scheduler["change_filter"]]
    86         kls = globals()[scheduler.pop('type')]
    87         # Python 2.6 can't handle unicode keys as keyword arguments:
    88         # http://bugs.python.org/issue2646.  Modern versions of json return
    89         # unicode strings from json.load, so we map all keys to str objects.
    90         scheduler = dict(map(lambda key_value_pair: (str(key_value_pair[0]), key_value_pair[1]), scheduler.items()))
    91 
    92         c['schedulers'].append(kls(**scheduler))
    93 
    94     forceScheduler = ForceScheduler(
    95         name="force",
    96         builderNames=[str(builder['name']) for builder in config['builders']],
    97         reason=StringParameter(name="reason", default="", size=40),
    98 
    99         # Validate SVN revision: number or empty string
    100         revision=StringParameter(name="revision", default="", regex=re.compile(r'^(\d*)$')),
    101 
    102         # Disable default enabled input fields: branch, repository, project, additional properties
    103         branch=FixedParameter(name="branch"),
    104         repository=FixedParameter(name="repository"),
    105         project=FixedParameter(name="project"),
    106         properties=[BooleanParameter(name="is_clean", label="Force Clean build")]
    107     )
    108     c['schedulers'].append(forceScheduler)
    109 
    110     c['builders'] = []
    111     for builder in config['builders']:
    112         for slaveName in builder['slavenames']:
    113             for slave in config['slaves']:
    114                 if slave['name'] != slaveName or slave['platform'] == '*':
    115                     continue
    116 
    117                 if slave['platform'] != builder['platform']:
    118                     raise Exception, "Builder %r is for platform %r but has slave %r for platform %r!" % (builder['name'], builder['platform'], slave['name'], slave['platform'])
    119 
    120                 break
    121 
    122         platform = builder['platform']
    123 
    124         builderType = builder.pop('type')
    125         factory = globals()["%sFactory" % builderType]
    126         factorykwargs = {}
    127         for key in "platform", "configuration", "architectures", "triggers", "additionalArguments", "SVNMirror":
    128             value = builder.pop(key, None)
    129             if value:
    130                 factorykwargs[key] = value
    131 
    132         builder["factory"] = factory(**factorykwargs)
    133 
    134         if platform.startswith('mac'):
    135             builder["category"] = 'AppleMac'
    136         elif platform.startswith('ios'):
    137             builder['category'] = 'iOS'
    138         elif platform == 'win':
    139             builder["category"] = 'AppleWin'
    140         elif platform.startswith('gtk'):
    141             builder["category"] = 'GTK'
    142         elif platform.startswith('wpe'):
    143             builder["category"] = 'WPE'
    144         else:
    145             builder["category"] = 'misc'
    146 
    147         if (builder['category'] in ('AppleMac', 'AppleWin', 'iOS')) and builderType != 'Build':
    148             builder['nextBuild'] = pickLatestBuild
    149 
    150         c['builders'].append(builder)
    151 
    15246loadBuilderConfig(c)

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py

    r225334 r225433  
    2525from buildbot.status.builder import SUCCESS, FAILURE, WARNINGS, SKIPPED, EXCEPTION
    2626
    27 from factories import *
    28 
    2927from twisted.internet import defer
    3028

  • trunk/Tools/ChangeLog

    r225425 r225433  
     12017-12-01  Aakash Jain  <aakash_jain@apple.com>
     2
     3        [build.webkit.org] Move python code to load config from master.cfg in separate file
     4        https://bugs.webkit.org/show_bug.cgi?id=180278
     5
     6        Reviewed by Daniel Bates.
     7
     8        * BuildSlaveSupport/build.webkit.org-config/master.cfg: Moved appropriate code to loadConfig.py
     9        * BuildSlaveSupport/build.webkit.org-config/loadConfig.py: Ditto.
     10        * BuildSlaveSupport/build.webkit.org-config/steps.py: Removed extra import.
     11
    1122017-12-01  Brian Burg  <bburg@apple.com>
    213
Note: See TracChangeset for help on using the changeset viewer.