Changeset 252075 in webkit
- Timestamp:
- Nov 5, 2019 1:36:43 PM (4 years ago)
- Location:
- trunk/Tools
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/BuildSlaveSupport/ews-build/loadConfig.py
r250356 r252075 47 47 else: 48 48 passwords = json.load(open(os.path.join(master_prefix_path, 'passwords.json'))) 49 results_server_api_key = passwords.get('results-server-api-key') 50 if results_server_api_key: 51 os.environ['RESULTS_SERVER_API_KEY'] = results_server_api_key 49 52 50 53 checkWorkersAndBuildersForConsistency(config, config['workers'], config['builders']) -
trunk/Tools/BuildSlaveSupport/ews-build/steps.py
r252037 r252075 32 32 33 33 import json 34 import os 34 35 import re 35 36 import requests … … 40 41 WithProperties = properties.WithProperties 41 42 Interpolate = properties.Interpolate 43 RESULTS_WEBKIT_URL = 'https://results.webkit.org' 44 RESULTS_SERVER_API_KEY = 'RESULTS_SERVER_API_KEY' 42 45 43 46 … … 1002 1005 WithProperties('--%(configuration)s')] 1003 1006 1007 def __init__(self, **kwargs): 1008 shell.Test.__init__(self, logEnviron=False, **kwargs) 1009 1004 1010 def start(self): 1005 1011 self.log_observer = logobserver.BufferLogObserver(wantStderr=True) … … 1156 1162 name = 'run-layout-tests-without-patch' 1157 1163 1164 def start(self): 1165 self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) 1166 self.setCommand(self.command + 1167 ['--buildbot-master', EWS_URL.replace('https://', '').strip('/'), 1168 '--builder-name', self.getProperty('buildername'), 1169 '--build-number', self.getProperty('buildnumber'), 1170 '--buildbot-worker', self.getProperty('workername'), 1171 '--report', RESULTS_WEBKIT_URL]) 1172 return super(RunWebKitTestsWithoutPatch, self).start() 1173 1158 1174 def evaluateCommand(self, cmd): 1159 1175 rc = shell.Test.evaluateCommand(self, cmd) … … 1473 1489 def evaluateCommand(self, cmd): 1474 1490 return TestWithFailureCount.evaluateCommand(self, cmd) 1491 1492 def start(self): 1493 self.workerEnvironment[RESULTS_SERVER_API_KEY] = os.getenv(RESULTS_SERVER_API_KEY) 1494 self.setCommand(self.command + 1495 ['--buildbot-master', EWS_URL.replace('https://', '').strip('/'), 1496 '--builder-name', self.getProperty('buildername'), 1497 '--build-number', self.getProperty('buildnumber'), 1498 '--buildbot-worker', self.getProperty('workername'), 1499 '--report', RESULTS_WEBKIT_URL]) 1500 return super(RunAPITestsWithoutPatch, self).start() 1475 1501 1476 1502 -
trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py
r251927 r252075 41 41 PrintConfiguration, ReRunAPITests, ReRunJavaScriptCoreTests, ReRunWebKitTests, RunAPITests, RunAPITestsWithoutPatch, 42 42 RunBindingsTests, RunBuildWebKitOrgUnitTests, RunEWSBuildbotCheckConfig, RunEWSUnitTests, RunJavaScriptCoreTests, RunJavaScriptCoreTestsToT, RunWebKit1Tests, 43 RunWebKitPerlTests, RunWebKitPyTests, RunWebKitTests, TestWithFailureCount, Trigger, TransferToS3, UnApplyPatchIfRequired,43 RunWebKitPerlTests, RunWebKitPyTests, RunWebKitTests, RunWebKitTestsWithoutPatch, TestWithFailureCount, Trigger, TransferToS3, UnApplyPatchIfRequired, 44 44 UpdateWorkingDirectory, UploadBuiltProduct, UploadTestResults, ValidatePatch) 45 45 … … 1144 1144 ExpectShell(workdir='wkdir', 1145 1145 logfiles={'json': self.jsonFileName}, 1146 logEnviron=False, 1146 1147 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1147 1148 ) … … 1158 1159 ExpectShell(workdir='wkdir', 1159 1160 logfiles={'json': self.jsonFileName}, 1161 logEnviron=False, 1160 1162 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1161 1163 ) … … 1175 1177 ExpectShell(workdir='wkdir', 1176 1178 logfiles={'json': self.jsonFileName}, 1179 logEnviron=False, 1177 1180 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1178 1181 ) … … 1203 1206 ExpectShell(workdir='wkdir', 1204 1207 logfiles={'json': self.jsonFileName}, 1208 logEnviron=False, 1205 1209 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1206 1210 ) … … 1221 1225 ExpectShell(workdir='wkdir', 1222 1226 logfiles={'json': self.jsonFileName}, 1227 logEnviron=False, 1223 1228 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1224 1229 ) … … 1239 1244 ExpectShell(workdir='wkdir', 1240 1245 logfiles={'json': self.jsonFileName}, 1246 logEnviron=False, 1241 1247 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1242 1248 ) … … 1257 1263 ExpectShell(workdir='wkdir', 1258 1264 logfiles={'json': self.jsonFileName}, 1265 logEnviron=False, 1259 1266 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--debug', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1260 1267 ) … … 1272 1279 ExpectShell(workdir='wkdir', 1273 1280 logfiles={'json': self.jsonFileName}, 1281 logEnviron=False, 1274 1282 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1275 1283 ) … … 1288 1296 1289 1297 1298 class TestRunWebKitTestsWithoutPatch(BuildStepMixinAdditions, unittest.TestCase): 1299 def setUp(self): 1300 self.longMessage = True 1301 self.jsonFileName = 'layout-test-results/full_results.json' 1302 return self.setUpBuildStep() 1303 1304 def tearDown(self): 1305 return self.tearDownBuildStep() 1306 1307 def configureStep(self): 1308 self.setupStep(RunWebKitTestsWithoutPatch()) 1309 self.property_exceed_failure_limit = 'clean_tree_results_exceed_failure_limit' 1310 self.property_failures = 'clean_tree_run_failures' 1311 self.setProperty('buildername', 'iOS-13-Simulator-WK2-Tests-EWS') 1312 self.setProperty('buildnumber', '123') 1313 self.setProperty('workername', 'ews126') 1314 os.environ['RESULTS_SERVER_API_KEY'] = 'sample-key' 1315 1316 def test_success(self): 1317 self.configureStep() 1318 self.setProperty('fullPlatform', 'ios-simulator') 1319 self.setProperty('configuration', 'release') 1320 self.expectRemoteCommands( 1321 ExpectShell(workdir='wkdir', 1322 logfiles={'json': self.jsonFileName}, 1323 env={'RESULTS_SERVER_API_KEY': 'sample-key'}, 1324 logEnviron=False, 1325 command=['python', 1326 'Tools/Scripts/run-webkit-tests', 1327 '--no-build', 1328 '--no-show-results', 1329 '--no-new-test-results', 1330 '--clobber-old-results', 1331 '--exit-after-n-failures', '30', 1332 '--skip-failing-tests', 1333 '--release', 1334 '--buildbot-master', 'ews-build.webkit.org', 1335 '--builder-name', 'iOS-13-Simulator-WK2-Tests-EWS', 1336 '--build-number', '123', 1337 '--buildbot-worker', 'ews126', 1338 '--report', 'https://results.webkit.org', 1339 '--results-directory', 'layout-test-results', 1340 '--debug-rwt-logging'], 1341 ) 1342 + 0, 1343 ) 1344 self.expectOutcome(result=SUCCESS, state_string='layout-tests') 1345 return self.runStep() 1346 1347 def test_failure(self): 1348 self.configureStep() 1349 self.setProperty('fullPlatform', 'ios-simulator') 1350 self.setProperty('configuration', 'release') 1351 self.expectRemoteCommands( 1352 ExpectShell(workdir='wkdir', 1353 logfiles={'json': self.jsonFileName}, 1354 env={'RESULTS_SERVER_API_KEY': 'sample-key'}, 1355 logEnviron=False, 1356 command=['python', 1357 'Tools/Scripts/run-webkit-tests', 1358 '--no-build', 1359 '--no-show-results', 1360 '--no-new-test-results', 1361 '--clobber-old-results', 1362 '--exit-after-n-failures', '30', 1363 '--skip-failing-tests', 1364 '--release', 1365 '--buildbot-master', 'ews-build.webkit.org', 1366 '--builder-name', 'iOS-13-Simulator-WK2-Tests-EWS', 1367 '--build-number', '123', 1368 '--buildbot-worker', 'ews126', 1369 '--report', 'https://results.webkit.org', 1370 '--results-directory', 'layout-test-results', 1371 '--debug-rwt-logging'], 1372 ) 1373 + ExpectShell.log('stdio', stdout='9 failures found.') 1374 + 2, 1375 ) 1376 self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)') 1377 return self.runStep() 1378 1379 1290 1380 class TestRunWebKit1Tests(BuildStepMixinAdditions, unittest.TestCase): 1291 1381 def setUp(self): … … 1304 1394 ExpectShell(workdir='wkdir', 1305 1395 logfiles={'json': self.jsonFileName}, 1396 logEnviron=False, 1306 1397 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--debug', '--dump-render-tree', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1307 1398 ) … … 1318 1409 ExpectShell(workdir='wkdir', 1319 1410 logfiles={'json': self.jsonFileName}, 1411 logEnviron=False, 1320 1412 command=['python', 'Tools/Scripts/run-webkit-tests', '--no-build', '--no-show-results', '--no-new-test-results', '--clobber-old-results', '--exit-after-n-failures', '30', '--skip-failing-tests', '--release', '--dump-render-tree', '--results-directory', 'layout-test-results', '--debug-rwt-logging'], 1321 1413 ) … … 2056 2148 return self.runStep() 2057 2149 2150 2151 class TestRunAPITestsWithoutPatch(BuildStepMixinAdditions, unittest.TestCase): 2152 def setUp(self): 2153 self.longMessage = True 2154 self.jsonFileName = 'api_test_results.json' 2155 return self.setUpBuildStep() 2156 2157 def tearDown(self): 2158 return self.tearDownBuildStep() 2159 2160 def test_success_mac(self): 2161 self.setupStep(RunAPITestsWithoutPatch()) 2162 self.setProperty('fullPlatform', 'mac-mojave') 2163 self.setProperty('platform', 'mac') 2164 self.setProperty('configuration', 'release') 2165 self.setProperty('buildername', 'API-Tests-macOS-EWS') 2166 self.setProperty('buildnumber', '11525') 2167 self.setProperty('workername', 'ews155') 2168 os.environ['RESULTS_SERVER_API_KEY'] = 'sample-key' 2169 2170 self.expectRemoteCommands( 2171 ExpectShell(workdir='wkdir', 2172 logEnviron=False, 2173 env={'RESULTS_SERVER_API_KEY': 'sample-key'}, 2174 command=['python', 2175 'Tools/Scripts/run-api-tests', 2176 '--no-build', 2177 '--release', 2178 '--verbose', 2179 '--json-output={0}'.format(self.jsonFileName), 2180 '--buildbot-master', 'ews-build.webkit.org', 2181 '--builder-name', 'API-Tests-macOS-EWS', 2182 '--build-number', '11525', 2183 '--buildbot-worker', 'ews155', 2184 '--report', 'https://results.webkit.org'], 2185 logfiles={'json': self.jsonFileName}, 2186 ) 2187 + ExpectShell.log('stdio', stdout='''... 2188 worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed 2189 worker/0 TestWTF.WTF_Variant.Ref Passed 2190 worker/0 TestWTF.WTF_Variant.RefPtr Passed 2191 worker/0 TestWTF.WTF_Variant.RetainPtr Passed 2192 worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed 2193 worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed 2194 Ran 1888 tests of 1888 with 1888 successful 2195 ------------------------------ 2196 All tests successfully passed! 2197 ''') 2198 + 0, 2199 ) 2200 self.expectOutcome(result=SUCCESS, state_string='run-api-tests-without-patch') 2201 return self.runStep() 2202 2203 def test_one_failure(self): 2204 self.setupStep(RunAPITestsWithoutPatch()) 2205 self.setProperty('fullPlatform', 'mac-mojave') 2206 self.setProperty('platform', 'ios-simulator') 2207 self.setProperty('configuration', 'debug') 2208 self.setProperty('buildername', 'API-Tests-iOS-EWS') 2209 self.setProperty('buildnumber', '123') 2210 self.setProperty('workername', 'ews156') 2211 2212 self.expectRemoteCommands( 2213 ExpectShell(workdir='wkdir', 2214 logEnviron=False, 2215 env={'RESULTS_SERVER_API_KEY': None}, 2216 command=['python', 2217 'Tools/Scripts/run-api-tests', 2218 '--no-build', 2219 '--debug', 2220 '--verbose', 2221 '--json-output={0}'.format(self.jsonFileName), 2222 '--buildbot-master', 'ews-build.webkit.org', 2223 '--builder-name', 'API-Tests-iOS-EWS', 2224 '--build-number', '123', 2225 '--buildbot-worker', 'ews156', 2226 '--report', 'https://results.webkit.org'], 2227 logfiles={'json': self.jsonFileName}, 2228 ) 2229 + ExpectShell.log('stdio', stdout=''' 2230 worker/0 TestWTF.WTF_Variant.OperatorAmpersand Passed 2231 worker/0 TestWTF.WTF_Variant.Ref Passed 2232 worker/0 TestWTF.WTF_Variant.RefPtr Passed 2233 worker/0 TestWTF.WTF_Variant.RetainPtr Passed 2234 worker/0 TestWTF.WTF_Variant.VisitorUsingMakeVisitor Passed 2235 worker/0 TestWTF.WTF_Variant.VisitorUsingSwitchOn Passed 2236 worker/0 exiting 2237 Ran 1888 tests of 1888 with 1887 successful 2238 ------------------------------ 2239 Test suite failed 2240 2241 Crashed 2242 2243 TestWTF.WTF.StringConcatenate_Unsigned 2244 **FAIL** WTF.StringConcatenate_Unsigned 2245 2246 Tools\\TestWebKitAPI\\Tests\\WTF\\StringConcatenate.cpp:84 2247 Value of: makeString('hello ', static_cast<unsigned short>(42) , ' world') 2248 Actual: hello 42 world 2249 Expected: 'hello * world' 2250 Which is: 74B00C9C 2251 2252 Testing completed, Exit status: 3 2253 ''') 2254 + 1, 2255 ) 2256 self.expectOutcome(result=FAILURE, state_string='1 api test failed or timed out') 2257 return self.runStep() 2058 2258 2059 2259 class TestArchiveTestResults(BuildStepMixinAdditions, unittest.TestCase): -
trunk/Tools/ChangeLog
r252062 r252075 1 2019-11-05 Aakash Jain <aakash_jain@apple.com> 2 3 EWS should report test failures from clean-tree to results.webkit.org 4 https://bugs.webkit.org/show_bug.cgi?id=203829 5 6 Reviewed by Jonathan Bedard. 7 8 * BuildSlaveSupport/ews-build/loadConfig.py: 9 (loadBuilderConfig): 10 * BuildSlaveSupport/ews-build/steps.py: 11 (RunWebKitTests.__init__): 12 (RunWebKitTestsWithoutPatch.start): 13 (RunAPITestsWithoutPatch.start): 14 * BuildSlaveSupport/ews-build/steps_unittest.py: Added and updated unit-tests. 15 1 16 2019-11-05 Wenson Hsieh <wenson_hsieh@apple.com> 2 17
Note: See TracChangeset
for help on using the changeset viewer.