Changeset 190764 in webkit
- Timestamp:
- Oct 8, 2015 6:13:26 PM (8 years ago)
- Location:
- trunk/Websites/perf.webkit.org
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Websites/perf.webkit.org/ChangeLog
r190704 r190764 1 2015-10-08 Ryosuke Niwa <rniwa@webkit.org> 2 3 pull-svn.py fails to sync revisions when SVN credentials is not setup 4 https://bugs.webkit.org/show_bug.cgi?id=149941 5 6 Reviewed by Chris Dumez. 7 8 Added the support for specifying subversion credentials. 9 10 Also added the support for pulling from multiple subversion servers. Subversion servers are specified 11 in a JSON configuration file specified by --svn-config formatted as follows: 12 13 [ 14 { 15 "name": "WebKit", 16 "url": "http://svn.webkit.org/repository/webkit", 17 "username": "webkitten", 18 "password": "webkitten's password", 19 "trustCertificate": true, 20 "accountNameFinderScript": 21 ["python", "/Volumes/Data/WebKit/Tools/Scripts/webkit-patch", "find-users"] 22 }, 23 ... 24 ] 25 26 In addition, refactored it to use the shared server config JSON for the dashboard access. 27 28 * tools/pull-svn.py: 29 (main): Now takes --svn-config-json, --server-config-json, --seconds-to-sleep and --max-fetch-count 30 as required options instead of seven unnamed arguments. 31 (fetch_commits_and_submit): Extracted from main. Fetches at most max_fetch_count new revisions from 32 the subversion server, and submits them in accordance with server_config. 33 (fetch_commit_and_resolve_author): Now takes a single repository dictionary instead of two separate 34 arguments for name and URL to pass down the repository's authentication info to fetch_commit. 35 (fetch_commit): Ditto. Add appropriate arguments when username and passwords are specified. 36 (resolve_author_name_from_account): Use a list argument instead of a single string argument now that 37 the argument comes from a JSON instead of sys.argv. 38 1 39 2015-10-07 Ryosuke Niwa <rniwa@webkit.org> 2 40 -
trunk/Websites/perf.webkit.org/tools/pull-svn.py
r185539 r190764 1 1 #!/usr/bin/python 2 2 3 import argparse 3 4 import json 4 5 import re … … 9 10 10 11 from xml.dom.minidom import parseString as parseXmlString 12 from util import setup_auth 11 13 from util import submit_commits 12 14 from util import text_content … … 14 16 15 17 def main(argv): 16 if len(argv) < 7: 17 sys.exit('Usage: pull-svn <repository-name> <repository-URL> <dashboard-URL> <slave-name> <slave-password> <seconds-to-sleep> [<account-to-name-helper>]') 18 parser = argparse.ArgumentParser() 19 parser.add_argument('--svn-config-json', required=True, help='The path to a JSON file that specifies subversion syncing options') 20 parser.add_argument('--server-config-json', required=True, help='The path to a JSON file that specifies the perf dashboard') 21 parser.add_argument('--seconds-to-sleep', type=float, default=900, help='The seconds to sleep between iterations') 22 parser.add_argument('--max-fetch-count', type=int, default=10, help='The number of commits to fetch at once') 23 args = parser.parse_args() 18 24 19 repository_name = argv[1] 20 repository_url = argv[2] 21 dashboard_url = argv[3] 22 slave_name = argv[4] 23 slave_password = argv[5] 24 seconds_to_sleep = float(argv[6]) 25 account_to_name_helper = argv[7] if len(argv) > 7 else None 25 with open(args.server_config_json) as server_config_json: 26 server_config = json.load(server_config_json) 27 setup_auth(server_config['server']) 26 28 27 print "Submitting revision logs for %s at %s to %s" % (repository_name, repository_url, dashboard_url) 28 29 revision_to_fetch = determine_first_revision_to_fetch(dashboard_url, repository_name) 30 print "Start fetching commits at r%d" % revision_to_fetch 31 32 pending_commits_to_send = [] 29 with open(args.svn_config_json) as svn_config_json: 30 svn_config = json.load(svn_config_json) 33 31 34 32 while True: 35 commit = fetch_commit_and_resolve_author(repository_name, repository_url, account_to_name_helper, revision_to_fetch) 33 for repository_info in svn_config: 34 fetch_commits_and_submit(repository_info, server_config, args.max_fetch_count) 35 print "Sleeping for %d seconds..." % args.seconds_to_sleep 36 time.sleep(args.seconds_to_sleep) 36 37 37 if commit:38 print "Fetched r%d." % revision_to_fetch39 pending_commits_to_send += [commit]40 revision_to_fetch += 141 else:42 print "Revision %d not found" % revision_to_fetch43 38 44 if not commit or len(pending_commits_to_send) >= 10: 45 if pending_commits_to_send: 46 print "Submitting the above commits to %s..." % dashboard_url 47 submit_commits(pending_commits_to_send, dashboard_url, slave_name, slave_password) 48 print "Successfully submitted." 49 pending_commits_to_send = [] 50 time.sleep(seconds_to_sleep) 39 def fetch_commits_and_submit(repository, server_config, max_fetch_count): 40 assert 'name' in repository, 'The repository name should be specified' 41 assert 'url' in repository, 'The SVN repository URL should be specified' 42 43 if 'revisionToFetch' not in repository: 44 print "Determining the stating revision for %s" % repository['name'] 45 repository['revisionToFecth'] = determine_first_revision_to_fetch(server_config['server']['url'], repository['name']) 46 47 pending_commits = [] 48 for unused in range(max_fetch_count): 49 commit = fetch_commit_and_resolve_author(repository, repository.get('accountNameFinderScript', None), repository['revisionToFecth']) 50 if not commit: 51 break 52 pending_commits += [commit] 53 repository['revisionToFecth'] += 1 54 55 if not pending_commits: 56 print "No new revision found for %s (waiting for r%d)" % (repository['name'], repository['revisionToFecth']) 57 return 58 59 revision_list = 'r' + ', r'.join(map(lambda commit: str(commit['revision']), pending_commits)) 60 print "Submitting revisions %s for %s to %s" % (revision_list, repository['name'], server_config['server']['url']) 61 submit_commits(pending_commits, server_config['server']['url'], server_config['slave']['name'], server_config['slave']['password']) 62 print "Successfully submitted." 63 print 51 64 52 65 … … 77 90 78 91 79 def fetch_commit_and_resolve_author(repository _name, repository_url, account_to_name_helper, revision_to_fetch):92 def fetch_commit_and_resolve_author(repository, account_to_name_helper, revision_to_fetch): 80 93 try: 81 commit = fetch_commit(repository _name, repository_url, revision_to_fetch)94 commit = fetch_commit(repository, revision_to_fetch) 82 95 except Exception as error: 83 96 sys.exit('Failed to fetch the commit %d: %s' % (revision_to_fetch, str(error))) … … 97 110 98 111 99 def fetch_commit(repository_name, repository_url, revision): 100 args = ['svn', 'log', '--revision', str(revision), '--xml', repository_url] 112 def fetch_commit(repository, revision): 113 args = ['svn', 'log', '--revision', str(revision), '--xml', repository['url'], '--non-interactive'] 114 if 'username' in repository and 'password' in repository: 115 args += ['--no-auth-cache', '--username', repository['username'], '--password', repository['password']] 116 if repository.get('trustCertificate', False): 117 args += ['--trust-server-cert'] 118 101 119 try: 102 120 output = subprocess.check_output(args, stderr=subprocess.STDOUT) … … 110 128 message = text_content(xml.getElementsByTagName("msg")[0]) 111 129 return { 112 'repository': repository _name,130 'repository': repository['name'], 113 131 'revision': revision, 114 132 'time': time, … … 122 140 123 141 def resolve_author_name_from_account(helper, account): 124 output = subprocess.check_output(helper + ' ' + account, shell=True)142 output = subprocess.check_output(helper + [account]) 125 143 match = name_account_compound_regex.match(output) 126 144 if match:
Note: See TracChangeset
for help on using the changeset viewer.