Changeset 231912 in webkit
- Timestamp:
- May 17, 2018 11:29:00 AM (6 years ago)
- Location:
- trunk/Tools
- Files:
-
- 1 added
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r231908 r231912 1 2018-05-17 Brendan McLoughlin <brendan@bocoup.com> 2 3 Export changes to web-platform-test as part of the webkit-patch upload workflow 4 https://bugs.webkit.org/show_bug.cgi?id=184914 5 6 Reviewed by Youenn Fablet. 7 8 * Scripts/webkitpy/tool/commands/upload.py: 9 (Upload): 10 (CreateBug.execute): 11 (WPTChangeExport): 12 * Scripts/webkitpy/tool/steps/__init__.py: 13 * Scripts/webkitpy/tool/steps/wptchangeexport.py: Added. 14 (WPTChangeExport): 15 (WPTChangeExport.options): 16 (WPTChangeExport.run): 17 * Scripts/webkitpy/w3c/common.py: 18 * Scripts/webkitpy/w3c/test_exporter.py: 19 (WebPlatformTestExporter): 20 (WebPlatformTestExporter.__init__): 21 (WebPlatformTestExporter.username): 22 (WebPlatformTestExporter.token): 23 (WebPlatformTestExporter._github): 24 (WebPlatformTestExporter._wpt_fork_branch_github_url): 25 (WebPlatformTestExporter._wpt_fork_remote): 26 (WebPlatformTestExporter._wpt_fork_push_url): 27 (WebPlatformTestExporter._git): 28 (WebPlatformTestExporter._branch_name): 29 (WebPlatformTestExporter._public_branch_name): 30 (WebPlatformTestExporter.has_wpt_changes): 31 (WebPlatformTestExporter._create_patch): 32 (WebPlatformTestExporter.create_git_patch): 33 (WebPlatformTestExporter._prompt_for_token): 34 (WebPlatformTestExporter._prompt_for_username): 35 (WebPlatformTestExporter._ensure_username_and_token): 36 (WebPlatformTestExporter._validate_and_save_token): 37 (WebPlatformTestExporter.push_to_wpt_fork): 38 (WebPlatformTestExporter.make_pull_request): 39 (WebPlatformTestExporter.delete_local_branch): 40 (parse_args): 41 (TestExporter): Deleted. 42 (TestExporter.__init__): Deleted. 43 (TestExporter._ensure_wpt_repository): Deleted. 44 (TestExporter._fetch_wpt_repository): Deleted. 45 (TestExporter._ensure_new_branch_name): Deleted. 46 (TestExporter.download_and_commit_patch): Deleted. 47 (TestExporter.clean): Deleted. 48 (TestExporter.create_branch_with_patch): Deleted. 49 (TestExporter.push_to_wpt_fork): Deleted. 50 (TestExporter.make_pull_request): Deleted. 51 (TestExporter.create_wpt_pull_request): Deleted. 52 (TestExporter.delete_local_branch): Deleted. 53 (TestExporter.create_git_patch): Deleted. 54 (TestExporter.create_upload_remote_if_needed): Deleted. 55 (TestExporter.do_export): Deleted. 56 * Scripts/webkitpy/w3c/test_exporter_unittest.py: 57 (TestExporterTest.test_export): 58 (TestExporterTest.test_export_with_specific_branch): 59 (TestExporterTest): 60 (TestExporterTest.test_export_interactive_mode): 61 (TestExporterTest.test_export_invalid_token): 62 (TestExporterTest.test_export_wrong_token): 63 1 64 2018-05-17 Valerie R Young <valerie@bocoup.com> 2 65 -
trunk/Tools/Scripts/webkitpy/tool/commands/upload.py
r225698 r231912 287 287 steps.PostDiff, 288 288 steps.SubmitToEWS, 289 steps.WPTChangeExport, 289 290 ] 290 291 long_help = """upload uploads the current diff to bugs.webkit.org. … … 532 533 else: 533 534 self.create_bug_from_patch(options, args, tool) 535 536 537 class WPTChangeExport(AbstractPatchUploadingCommand): 538 name = "wpt-change-export" 539 help_text = "Opens a pull request to synchronize any changes in the LayoutTests/imported/w3c/web-platform-tests directory" 540 argument_names = "[BUGID]" 541 steps = [ 542 steps.WPTChangeExport, 543 ] 544 545 long_help = """Opens a pull request to the w3c/web-platform-tests 546 github repo for any changes in the 547 LayoutTests/imported/w3c/web-platform-tests directory. This step 548 will noop if there are no changes in the web-platform-tests directory. 549 The user will be prompted to provide a github username and OAuth token 550 the first time this is run. 551 """ 552 553 def _prepare_state(self, options, args, tool): 554 state = {} 555 state["bug_id"] = self._bug_id(options, args, tool, state) 556 return state -
trunk/Tools/Scripts/webkitpy/tool/steps/__init__.py
r220715 r231912 67 67 from webkitpy.tool.steps.validatechangelogs import ValidateChangeLogs 68 68 from webkitpy.tool.steps.validatereviewer import ValidateReviewer 69 from webkitpy.tool.steps.wptchangeexport import WPTChangeExport -
trunk/Tools/Scripts/webkitpy/w3c/common.py
r226009 r231912 31 31 import json 32 32 import logging 33 import os 33 34 34 35 35 WPT_GH_ORG = 'w3c'36 WPT_GH_REPO_NAME = 'web-platform-tests'36 WPT_GH_ORG = os.environ.get('WPT_GH_ORG', 'w3c') 37 WPT_GH_REPO_NAME = os.environ.get('WPT_GH_REPO_NAME', 'web-platform-tests') 37 38 WPT_GH_URL = 'https://github.com/%s/%s/' % (WPT_GH_ORG, WPT_GH_REPO_NAME) 38 39 WPT_MIRROR_URL = 'https://chromium.googlesource.com/external/w3c/web-platform-tests.git' -
trunk/Tools/Scripts/webkitpy/w3c/test_exporter.py
r229921 r231912 29 29 import os 30 30 import time 31 import json 32 from urllib2 import HTTPError 31 33 32 34 from webkitpy.common.checkout.scm.git import Git … … 36 38 from webkitpy.w3c.wpt_github import WPTGitHub 37 39 from webkitpy.w3c.wpt_linter import WPTLinter 40 from webkitpy.w3c.common import WPT_GH_ORG 41 from webkitpy.common.memoized import memoized 38 42 39 43 _log = logging.getLogger(__name__) 40 44 41 45 WEBKIT_WPT_DIR = 'LayoutTests/imported/w3c/web-platform-tests' 42 WPT_PR_URL = "https://github.com/ w3c/web-platform-tests/pull/"46 WPT_PR_URL = "https://github.com/%s/web-platform-tests/pull/" % WPT_GH_ORG 43 47 WEBKIT_EXPORT_PR_LABEL = 'webkit-export' 44 48 45 49 46 class TestExporter(object): 47 50 class WebPlatformTestExporter(object): 48 51 def __init__(self, host, options, gitClass=Git, bugzillaClass=Bugzilla, WPTGitHubClass=WPTGitHub, WPTLinterClass=WPTLinter): 49 52 self._host = host … … 53 56 self._host.initialize_scm() 54 57 58 self._WPTGitHubClass = WPTGitHubClass 59 self._gitClass = gitClass 55 60 self._bugzilla = bugzillaClass() 56 61 self._bug_id = options.bug_id … … 65 70 self._options.repository_directory = webkit_finder.path_from_webkit_base('WebKitBuild', 'w3c-tests', 'web-platform-tests') 66 71 67 self._git = self._ensure_wpt_repository("https://github.com/w3c/web-platform-tests.git", self._options.repository_directory, gitClass)68 72 self._linter = WPTLinterClass(self._options.repository_directory, host.filesystem) 69 73 74 self._bugzilla_url = "https://bugs.webkit.org/show_bug.cgi?id=" + str(self._bug_id) 75 self._commit_message = options.message 76 if not self._commit_message: 77 self._commit_message = 'WebKit export of ' + self._bugzilla_url if self._bug_id else 'Export made from a WebKit repository' 78 79 @property 80 def username(self): 81 if hasattr(self, '_username'): 82 return self._username 83 84 self._ensure_username_and_token(self._options) 85 return self._username 86 87 @property 88 def token(self): 89 if hasattr(self, '_token'): 90 return self._token 91 92 self._ensure_username_and_token(self._options) 93 return self._token 94 95 @property 96 @memoized 97 def _github(self): 98 return self._WPTGitHubClass(self._host, self.username, self.token) if self.username and self.token else None 99 100 @property 101 @memoized 102 def _wpt_fork_branch_github_url(self): 103 return "https://github.com/" + self.username + "/web-platform-tests/tree/" + self._public_branch_name 104 105 @property 106 @memoized 107 def _wpt_fork_remote(self): 108 wpt_fork_remote = self._options.repository_remote 109 if not wpt_fork_remote: 110 wpt_fork_remote = self.username 111 112 return wpt_fork_remote 113 114 @property 115 @memoized 116 def _wpt_fork_push_url(self): 117 wpt_fork_push_url = self._options.repository_remote_url 118 if not wpt_fork_push_url: 119 wpt_fork_push_url = "https://" + self.username + "@github.com/" + self.username + "/web-platform-tests.git" 120 121 return wpt_fork_push_url 122 123 @property 124 @memoized 125 def _git(self): 126 return self._ensure_wpt_repository("https://github.com/w3c/web-platform-tests.git", self._options.repository_directory, self._gitClass) 127 128 @property 129 @memoized 130 def _branch_name(self): 131 return self._ensure_new_branch_name() 132 133 @property 134 @memoized 135 def _public_branch_name(self): 136 options = self._options 137 return options.public_branch_name if options.public_branch_name else self._branch_name 138 139 @property 140 @memoized 141 def _wpt_patch(self): 142 patch_data = self._host.scm().create_patch(self._options.git_commit, [WEBKIT_WPT_DIR]) 143 if not patch_data or not 'diff' in patch_data: 144 return '' 145 return patch_data 146 147 def has_wpt_changes(self): 148 return bool(self._wpt_patch) 149 150 def write_git_patch_file(self): 151 _, patch_file = self._filesystem.open_binary_tempfile('wpt_export_patch') 152 patch_data = self._wpt_patch 153 if not 'diff' in patch_data: 154 _log.info('No changes to upstream, patch data is: "%s"' % (patch_data)) 155 return '' 156 # FIXME: We can probably try to use --relative git parameter to not do that replacement. 157 patch_data = patch_data.replace(WEBKIT_WPT_DIR + '/', '') 158 self._filesystem.write_text_file(patch_file, patch_data) 159 return patch_file 160 161 def _prompt_for_token(self, options): 162 if options.non_interactive: 163 return None 164 return self._host.user.prompt_password('Enter GitHub OAuth token (or empty string to skip creating a pull request): ') 165 166 def _prompt_for_username(self, options): 167 if options.non_interactive: 168 return None 169 return self._host.user.prompt('Enter your GitHub username: ') 170 171 def _ensure_username_and_token(self, options): 70 172 self._username = options.username 71 173 if not self._username: 174 # FIXME: Use the keychain to store username and oauth token instead of .git/config 72 175 self._username = self._git.local_config('github.username').rstrip() 73 176 if not self._username: 74 177 self._username = os.environ.get('GITHUB_USERNAME') 75 178 if not self._username: 179 self._username = self._prompt_for_username(options) 180 if not self._username: 76 181 raise ValueError("Missing GitHub username, please provide it as a command argument (see help for the command).") 77 elif not self._git.local_config('github.username'):78 self._git.set_local_config('github.username', self._username)79 182 80 183 self._token = options.token … … 84 187 self._token = os.environ.get('GITHUB_TOKEN') 85 188 if not self._token: 189 self._token = self._prompt_for_token(options) 190 if not self._token: 86 191 _log.info("Missing GitHub token, the script will not be able to create a pull request to W3C web-platform-tests repository.") 87 elif not self._git.local_config('github.token'): 88 self._git.set_local_config('github.token', self._token) 89 90 self._github = WPTGitHubClass(self._host, self._username, self._token) if self._username and self._token else None 91 92 self._branch_name = self._ensure_new_branch_name() 93 self._public_branch_name = options.public_branch_name if options.public_branch_name else self._branch_name 94 self._bugzilla_url = "https://bugs.webkit.org/show_bug.cgi?id=" + str(self._bug_id) 95 self._commit_message = options.message 96 if not self._commit_message: 97 self._commit_message = 'WebKit export of ' + self._bugzilla_url if self._bug_id else 'Export made from a WebKit repository' 98 99 self._wpt_fork_remote = options.repository_remote 100 if not self._wpt_fork_remote: 101 self._wpt_fork_remote = self._username 102 103 self._wpt_fork_push_url = options.repository_remote_url 104 if not self._wpt_fork_push_url: 105 self._wpt_fork_push_url = "https://" + self._username + "@github.com/" + self._username + "/web-platform-tests.git" 192 193 if self._token: 194 self._validate_and_save_token(self._username, self._token) 195 196 def _validate_and_save_token(self, username, token): 197 url = 'https://api.github.com/user?access_token=%s' % (token,) 198 try: 199 response = self._host.web.request(method='GET', url=url, data=None) 200 except HTTPError as e: 201 raise Exception("OAuth token is not valid") 202 data = json.load(response) 203 login = data.get('login', None) 204 if login != username: 205 raise Exception("OAuth token does not match the provided username. Provided user: %s, github login: %s" % (username, login)) 206 else: 207 # Username and token are valid. Save them in the git config so we 208 # do not need to ask for them again 209 if not self._git.local_config('github.token'): 210 self._git.set_local_config('github.token', token) 211 if not self._git.local_config('github.username'): 212 self._git.set_local_config('github.username', username) 106 213 107 214 def _ensure_wpt_repository(self, url, wpt_repository_directory, gitClass): … … 167 274 def push_to_wpt_fork(self): 168 275 self.create_upload_remote_if_needed() 169 wpt_fork_branch_github_url = "https://github.com/" + self._username + "/web-platform-tests/tree/" + self._public_branch_name170 276 _log.info('Pushing branch ' + self._branch_name + " to " + self._git.remote(["get-url", self._wpt_fork_remote]).rstrip()) 171 277 _log.info('This may take some time') 172 278 self._git.push([self._wpt_fork_remote, self._branch_name + ":" + self._public_branch_name, '-f']) 173 _log.info('Branch available at ' + wpt_fork_branch_github_url)279 _log.info('Branch available at ' + self._wpt_fork_branch_github_url) 174 280 return True 175 281 176 282 def make_pull_request(self): 177 283 if not self._github: 178 _log.info(' Missing information to create a pull request')284 _log.info('Skipping pull request because OAuth token was not provided. You can open the pull request manually using the branch ' + self._wpt_fork_branch_github_url) 179 285 return 180 286 … … 187 293 except Exception as e: 188 294 _log.warning(e) 189 _log.info('Could not add label "%s" to pr #%s. User "%s" may not have permission to update labels in the w3c/web-platform-test repo.' % (WEBKIT_EXPORT_PR_LABEL, pr_number, self. _username))295 _log.info('Could not add label "%s" to pr #%s. User "%s" may not have permission to update labels in the w3c/web-platform-test repo.' % (WEBKIT_EXPORT_PR_LABEL, pr_number, self.username)) 190 296 if self._bug_id and pr_number: 191 297 self._bugzilla.post_comment_to_bug(self._bug_id, "Submitted web-platform-tests pull request: " + WPT_PR_URL + str(pr_number)) … … 204 310 205 311 def delete_local_branch(self): 206 _log.info('Removing branch ' + self._branch_name)312 _log.info('Removing local branch ' + self._branch_name) 207 313 self._git.checkout('master') 208 314 self._git.delete_branch(self._branch_name) 209 210 def create_git_patch(self):211 patch_file = './patch.temp.' + str(time.clock())212 git_commit = "HEAD...." if not self._options.git_commit else self._options.git_commit + "~1.." + self._options.git_commit213 patch_data = self._host.scm().create_patch(git_commit, [WEBKIT_WPT_DIR])214 if not patch_data or not 'diff' in patch_data:215 _log.info('No changes to upstream, patch data is: "%s"' % (patch_data))216 return ''217 # FIXME: We can probably try to use --relative git parameter to not do that replacement.218 patch_data = patch_data.replace(WEBKIT_WPT_DIR + '/', '')219 patch_file = self._filesystem.abspath(patch_file)220 self._filesystem.write_text_file(patch_file, patch_data)221 return patch_file222 315 223 316 def create_upload_remote_if_needed(self): … … 226 319 227 320 def do_export(self): 228 git_patch_file = self. create_git_patch()321 git_patch_file = self.write_git_patch_file() 229 322 230 323 if not git_patch_file: … … 291 384 parser.add_argument('-d', '--repository', dest='repository_directory', default=None, help='repository directory') 292 385 parser.add_argument('-c', '--create-pr', dest='create_pull_request', action='store_true', default=False, help='create pull request to w3c web-platform-tests') 386 parser.add_argument('--non-interactive', action='store_true', dest='non_interactive', default=False, help='Never prompt the user, fail as fast as possible.') 293 387 294 388 options, args = parser.parse_known_args(args) … … 306 400 307 401 logger = logging.getLogger('webkitpy.w3c.test_exporter') 402 logger.propagate = False 308 403 logger.setLevel(logging.INFO) 309 404 handler = LogHandler() … … 317 412 318 413 configure_logging() 319 320 test_exporter = TestExporter(Host(), options) 414 test_exporter = WebPlatformTestExporter(Host(), options) 415 416 if not test_exporter.has_wpt_changes(): 417 _log.info('No changes to upstream. Exiting...') 418 return 321 419 322 420 test_exporter.do_export() -
trunk/Tools/Scripts/webkitpy/w3c/test_exporter_unittest.py
r229921 r231912 26 26 from webkitpy.common.system.filesystem_mock import MockFileSystem 27 27 from webkitpy.common.system.executive_mock import MockExecutive2 28 from webkitpy.w3c.test_exporter import TestExporter, parse_args28 from webkitpy.w3c.test_exporter import WebPlatformTestExporter, parse_args 29 29 from webkitpy.w3c.wpt_github_mock import MockWPTGitHub 30 30 … … 123 123 def test_export(self): 124 124 host = TestExporterTest.MyMockHost() 125 host.web.responses.append({'status_code': 200, 'body': '{"login": "USER"}'}) 125 126 options = parse_args(['test_exporter.py', '-g', 'HEAD', '-b', '1234', '-c', '-n', 'USER', '-t', 'TOKEN']) 126 exporter = TestExporter(host, options, TestExporterTest.MockGit, TestExporterTest.MockBugzilla, MockWPTGitHub, TestExporterTest.MockWPTLinter)127 exporter = WebPlatformTestExporter(host, options, TestExporterTest.MockGit, TestExporterTest.MockBugzilla, MockWPTGitHub, TestExporterTest.MockWPTLinter) 127 128 exporter.do_export() 128 129 self.assertEquals(exporter._github.calls, ['create_pr', 'add_label "webkit-export"']) … … 151 152 def test_export_with_specific_branch(self): 152 153 host = TestExporterTest.MyMockHost() 154 host.web.responses.append({'status_code': 200, 'body': '{"login": "USER"}'}) 153 155 options = parse_args(['test_exporter.py', '-g', 'HEAD', '-b', '1234', '-c', '-n', 'USER', '-t', 'TOKEN', '-bn', 'wpt-export-branch']) 154 exporter = TestExporter(host, options, TestExporterTest.MockGit, TestExporterTest.MockBugzilla, MockWPTGitHub, TestExporterTest.MockWPTLinter)156 exporter = WebPlatformTestExporter(host, options, TestExporterTest.MockGit, TestExporterTest.MockBugzilla, MockWPTGitHub, TestExporterTest.MockWPTLinter) 155 157 exporter.do_export() 156 158 self.assertEquals(exporter._git.calls, [ … … 170 172 'checkout master', 171 173 'reset hard origin/master']) 174 175 def test_export_interactive_mode(self): 176 host = TestExporterTest.MyMockHost() 177 host.web.responses.append({'status_code': 200, 'body': '{"login": "USER"}'}) 178 options = parse_args(['test_exporter.py', '-g', 'HEAD', '-b', '1234', '-c', '-n', 'USER', '-t', 'TOKEN', '--interactive']) 179 exporter = WebPlatformTestExporter(host, options, TestExporterTest.MockGit, TestExporterTest.MockBugzilla, MockWPTGitHub, TestExporterTest.MockWPTLinter) 180 exporter.do_export() 181 182 def test_export_invalid_token(self): 183 host = TestExporterTest.MyMockHost() 184 host.web.responses.append({'status_code': 401}) 185 options = parse_args(['test_exporter.py', '-g', 'HEAD', '-b', '1234', '-c', '-n', 'USER', '-t', 'TOKEN']) 186 exporter = WebPlatformTestExporter(host, options, TestExporterTest.MockGit, TestExporterTest.MockBugzilla, MockWPTGitHub, TestExporterTest.MockWPTLinter) 187 with self.assertRaises(Exception) as context: 188 exporter.do_export() 189 self.assertIn('OAuth token is not valid', str(context.exception)) 190 191 def test_export_wrong_token(self): 192 host = TestExporterTest.MyMockHost() 193 host.web.responses.append({'status_code': 200, 'body': '{"login": "DIFF_USER"}'}) 194 options = parse_args(['test_exporter.py', '-g', 'HEAD', '-b', '1234', '-c', '-n', 'USER', '-t', 'TOKEN']) 195 exporter = WebPlatformTestExporter(host, options, TestExporterTest.MockGit, TestExporterTest.MockBugzilla, MockWPTGitHub, TestExporterTest.MockWPTLinter) 196 with self.assertRaises(Exception) as context: 197 exporter.do_export() 198 self.assertIn('OAuth token does not match the provided username', str(context.exception))
Note: See TracChangeset
for help on using the changeset viewer.