Changeset 148349 in webkit
- Timestamp:
- Apr 13, 2013 12:40:07 AM (11 years ago)
- Location:
- trunk/Tools
- Files:
-
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r148328 r148349 1 2013-04-13 Zan Dobersek <zdobersek@igalia.com> 2 3 Fix flakiness dashboard unit test failures 4 https://bugs.webkit.org/show_bug.cgi?id=114518 5 6 Reviewed by Ryosuke Niwa. 7 8 Fix the current failures in unit tests for the flakiness dashboard by adjusting failing 9 test cases to simulate loading the builders list from the webkit.org build master, replacing 10 the Chromium builders and platforms that are used in tests with builders and platforms for the 11 Apple Mac Lion WebKit2 port, and removing two test cases that were testing Chromium-specific 12 functionality of the dashboard. 13 14 This is sort of an interlude to the flakiness dashboard cleanup, with the removal of Chromium-specific 15 code and hopefully general simplification of the dashboard codebase coming in near future. 16 17 * TestResultServer/static-dashboards/flakiness_dashboard_unittests.js: 18 (resetGlobals): 19 (test): 20 * TestResultServer/static-dashboards/loader_unittests.js: 21 1 22 2013-04-12 David Kilzer <ddkilzer@apple.com> 2 23 -
trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_unittests.js
r147457 r148349 45 45 46 46 LOAD_BUILDBOT_DATA([{ 47 name: 'ChromiumWebkit',48 url: 'dummyurl',49 tests: {'layout-tests': {'builders': ['WebKit Linux', 'WebKit Linux (dbg)', 'WebKit Mac10.7', 'WebKit Win', 'WebKit Win (dbg)']}}50 },51 {52 47 name: 'webkit.org', 53 48 url: 'dummyurl', 54 tests: {'layout-tests': {'builders': ['Apple SnowLeopard Tests', 'Qt Linux Tests', 'Chromium Mac10.7 Tests', 'GTK Win']}}49 tests: {'layout-tests': {'builders': ['Apple Lion Release WK2 (Tests)', 'Apple Lion Debug WK2 (Tests)', 'GTK Linux 64-bit Release', 'Qt Linux Tests']}} 55 50 }]); 56 51 … … 90 85 test('releaseFail', 2, function() { 91 86 resetGlobals(); 92 loadBuildersList('@ToT - chromium.org', 'layout-tests');93 94 var builder = ' WebKit Win';87 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 88 89 var builder = 'Apple Lion Release WK2 (Tests)'; 95 90 var test = 'foo/1.html'; 96 91 var expectationsArray = [ 97 92 {'modifiers': 'RELEASE', 'expectations': 'FAIL'} 98 93 ]; 99 g_expectationsByPlatform[' CHROMIUM'] = getParsedExpectations('[ Release ] ' + test + ' [ Failure ]');94 g_expectationsByPlatform['APPLE_MAC_LION_WK2'] = getParsedExpectations('[ Release ] ' + test + ' [ Failure ]'); 100 95 runExpectationsTest(builder, test, 'FAIL', 'RELEASE'); 101 96 }); … … 103 98 test('releaseFailDebugCrashReleaseBuilder', 2, function() { 104 99 resetGlobals(); 105 loadBuildersList('@ToT - chromium.org', 'layout-tests'); 106 var builder = 'WebKit Win'; 100 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 101 102 var builder = 'Apple Lion Release WK2 (Tests)'; 107 103 var test = 'foo/1.html'; 108 104 var expectationsArray = [ … … 110 106 {'modifiers': 'DEBUG', 'expectations': 'CRASH'} 111 107 ]; 112 g_expectationsByPlatform[' CHROMIUM'] = getParsedExpectations('[ Release ] ' + test + ' [ Failure ]\n' +108 g_expectationsByPlatform['APPLE_MAC_LION_WK2'] = getParsedExpectations('[ Release ] ' + test + ' [ Failure ]\n' + 113 109 '[ Debug ] ' + test + ' [ Crash ]'); 114 110 runExpectationsTest(builder, test, 'FAIL', 'RELEASE'); … … 117 113 test('releaseFailDebugCrashDebugBuilder', 2, function() { 118 114 resetGlobals(); 119 loadBuildersList('@ToT - chromium.org', 'layout-tests'); 120 var builder = 'WebKit Win (dbg)'; 115 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 116 117 var builder = 'Apple Lion Debug WK2 (Tests)'; 121 118 var test = 'foo/1.html'; 122 119 var expectationsArray = [ … … 124 121 {'modifiers': 'DEBUG', 'expectations': 'CRASH'} 125 122 ]; 126 g_expectationsByPlatform[' CHROMIUM'] = getParsedExpectations('[ Release ] ' + test + ' [ Failure ]\n' +123 g_expectationsByPlatform['APPLE_MAC_LION_WK2'] = getParsedExpectations('[ Release ] ' + test + ' [ Failure ]\n' + 127 124 '[ Debug ] ' + test + ' [ Crash ]'); 128 125 runExpectationsTest(builder, test, 'CRASH', 'DEBUG'); 129 126 }); 130 127 131 test('overrideJustBuildType', 12, function() {128 test('overrideJustBuildType', 4, function() { 132 129 resetGlobals(); 133 loadBuildersList('@ToT - chromium.org', 'layout-tests');130 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 134 131 var test = 'bar/1.html'; 135 g_expectationsByPlatform['CHROMIUM'] = getParsedExpectations('bar [ WontFix Failure Pass Timeout ]\n' + 136 '[ Mac ] ' + test + ' [ WontFix Failure ]\n' + 137 '[ Linux Debug ] ' + test + ' [ Crash ]'); 138 139 runExpectationsTest('WebKit Win', test, 'FAIL PASS TIMEOUT', 'WONTFIX'); 140 runExpectationsTest('WebKit Win (dbg)(3)', test, 'FAIL PASS TIMEOUT', 'WONTFIX'); 141 runExpectationsTest('WebKit Linux', test, 'FAIL PASS TIMEOUT', 'WONTFIX'); 142 runExpectationsTest('WebKit Linux (dbg)(3)', test, 'CRASH', 'LINUX DEBUG'); 143 runExpectationsTest('WebKit Mac10.7', test, 'FAIL', 'MAC WONTFIX'); 144 runExpectationsTest('WebKit Mac10.7 (dbg)(3)', test, 'FAIL', 'MAC WONTFIX'); 132 g_expectationsByPlatform['APPLE_MAC_LION_WK2'] = getParsedExpectations('bar [ WontFix Failure Pass Timeout ]\n' + 133 '[ Release ] ' + test + ' [ WontFix Failure ]\n' + 134 '[ Debug ] ' + test + ' [ Crash ]'); 135 136 runExpectationsTest('Apple Lion Release WK2 (Tests)', test, 'FAIL', 'RELEASE WONTFIX'); 137 runExpectationsTest('Apple Lion Debug WK2 (Tests)', test, 'CRASH', 'DEBUG'); 145 138 }); 146 139 … … 231 224 }); 232 225 233 test('getExpectations', 1 6, function() {226 test('getExpectations', 14, function() { 234 227 resetGlobals(); 235 loadBuildersList('@ToT - chromium.org', 'layout-tests');228 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 236 229 237 230 stubResultsByBuilder({ 238 ' WebKit Win' : {231 'Apple Lion Release WK2 (Tests)' : { 239 232 'tests': { 240 233 'foo/test1.html': {'results': [[100, 'F']], 'times': [[100, 0]]}, … … 246 239 }); 247 240 248 g_expectationsByPlatform[' CHROMIUM'] = getParsedExpectations('Bug(123) foo [ Failure Pass Crash ]\n' +241 g_expectationsByPlatform['APPLE_MAC_LION_WK2'] = getParsedExpectations('Bug(123) foo [ Failure Pass Crash ]\n' + 249 242 'Bug(Foo) [ Release ] foo/test1.html [ Failure ]\n' + 250 243 '[ Debug ] foo/test1.html [ Crash ]\n' + 251 244 'Bug(456) foo/test2.html [ Failure ]\n' + 252 '[ LinuxDebug ] foo/test2.html [ Crash ]\n' +245 '[ Debug ] foo/test2.html [ Crash ]\n' + 253 246 '[ Release ] test1.html [ Failure ]\n' + 254 247 '[ Debug ] test1.html [ Crash ]\n'); 255 g_expectationsByPlatform['CHROMIUM_ANDROID'] = getParsedExpectations('Bug(654) foo/test2.html [ Crash ]\n');256 248 257 249 g_expectationsByPlatform['GTK'] = getParsedExpectations('Bug(42) foo/test2.html [ ImageOnlyFailure ]\n' + … … 263 255 processExpectations(); 264 256 265 var expectations = getExpectations('foo/test1.html', ' CHROMIUM_XP', 'DEBUG');257 var expectations = getExpectations('foo/test1.html', 'APPLE_MAC_LION_WK2', 'DEBUG'); 266 258 equal(JSON.stringify(expectations), '{"modifiers":"DEBUG","expectations":"CRASH"}'); 267 259 268 var expectations = getExpectations('foo/test1.html', ' CHROMIUM_LUCID', 'RELEASE');260 var expectations = getExpectations('foo/test1.html', 'APPLE_MAC_LION_WK2', 'RELEASE'); 269 261 equal(JSON.stringify(expectations), '{"modifiers":"Bug(Foo) RELEASE","expectations":"FAIL"}'); 270 262 271 var expectations = getExpectations('foo/test2.html', ' CHROMIUM_LUCID', 'RELEASE');263 var expectations = getExpectations('foo/test2.html', 'APPLE_MAC_LION_WK2', 'RELEASE'); 272 264 equal(JSON.stringify(expectations), '{"modifiers":"Bug(456)","expectations":"FAIL"}'); 273 265 274 var expectations = getExpectations('foo/test2.html', 'CHROMIUM_LION', 'DEBUG'); 275 equal(JSON.stringify(expectations), '{"modifiers":"Bug(456)","expectations":"FAIL"}'); 276 277 var expectations = getExpectations('foo/test2.html', 'CHROMIUM_LUCID', 'DEBUG'); 278 equal(JSON.stringify(expectations), '{"modifiers":"LINUX DEBUG","expectations":"CRASH"}'); 279 280 var expectations = getExpectations('foo/test2.html', 'CHROMIUM_ANDROID', 'RELEASE'); 281 equal(JSON.stringify(expectations), '{"modifiers":"Bug(654)","expectations":"CRASH"}'); 282 283 var expectations = getExpectations('test1.html', 'CHROMIUM_ANDROID', 'RELEASE'); 266 var expectations = getExpectations('foo/test2.html', 'APPLE_MAC_LION_WK2', 'DEBUG'); 267 equal(JSON.stringify(expectations), '{"modifiers":"DEBUG","expectations":"CRASH"}'); 268 269 var expectations = getExpectations('test1.html', 'APPLE_MAC_LION_WK2', 'RELEASE'); 284 270 equal(JSON.stringify(expectations), '{"modifiers":"RELEASE","expectations":"FAIL"}'); 285 271 286 var expectations = getExpectations('foo/test3.html', ' CHROMIUM_LUCID', 'DEBUG');272 var expectations = getExpectations('foo/test3.html', 'APPLE_MAC_LION_WK2', 'DEBUG'); 287 273 equal(JSON.stringify(expectations), '{"modifiers":"Bug(123)","expectations":"FAIL PASS CRASH"}'); 288 274 289 var expectations = getExpectations('test1.html', ' CHROMIUM_XP', 'DEBUG');275 var expectations = getExpectations('test1.html', 'APPLE_MAC_LION_WK2', 'DEBUG'); 290 276 equal(JSON.stringify(expectations), '{"modifiers":"DEBUG","expectations":"CRASH"}'); 291 277 292 var expectations = getExpectations('test1.html', ' CHROMIUM_LUCID', 'RELEASE');278 var expectations = getExpectations('test1.html', 'APPLE_MAC_LION_WK2', 'RELEASE'); 293 279 equal(JSON.stringify(expectations), '{"modifiers":"RELEASE","expectations":"FAIL"}'); 294 280 … … 329 315 // FIXME(jparent): Remove this once global isn't used. 330 316 g_history = historyInstance; 331 loadBuildersList('@ToT - chromium.org', 'layout-tests');332 var builder = ' WebKit Win';317 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 318 var builder = 'Apple Lion Release WK2 (Tests)'; 333 319 g_perBuilderWithExpectationsButNoFailures[builder] = ['passing-test1.html', 'passing-test2.html']; 334 320 g_perBuilderSkippedPaths[builder] = ['skipped-test1.html']; … … 338 324 historyInstance.dashboardSpecificState.showSkipped = true; 339 325 340 historyInstance.crossDashboardState.group = '@ToT - chromium.org';326 historyInstance.crossDashboardState.group = '@ToT - webkit.org'; 341 327 historyInstance.crossDashboardState.testType = 'layout-tests'; 342 328 … … 372 358 var group = selects[1]; 373 359 equal(group.parentNode.textContent.indexOf('Group:'), 0); 374 equal(group.children.length, 3);360 equal(group.children.length, 1); 375 361 376 362 historyInstance.crossDashboardState.testType = 'layout-tests'; … … 380 366 var group = selects[1]; 381 367 equal(group.parentNode.textContent.indexOf('Group:'), 0); 382 equal(group.children.length, 4);368 equal(group.children.length, 1); 383 369 }); 384 370 385 371 test('htmlForIndividualTestOnAllBuilders', 1, function() { 386 372 resetGlobals(); 387 loadBuildersList('@ToT - chromium.org', 'layout-tests');373 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 388 374 equal(htmlForIndividualTestOnAllBuilders('foo/nonexistant.html'), '<div class="not-found">Test not found. Either it does not exist, is skipped or passes on all platforms.</div>'); 389 375 }); … … 391 377 test('htmlForIndividualTestOnAllBuildersWithResultsLinksNonexistant', 1, function() { 392 378 resetGlobals(); 393 loadBuildersList('@ToT - chromium.org', 'layout-tests');379 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 394 380 equal(htmlForIndividualTestOnAllBuildersWithResultsLinks('foo/nonexistant.html'), 395 381 '<div class="not-found">Test not found. Either it does not exist, is skipped or passes on all platforms.</div>' + … … 404 390 405 391 test('htmlForIndividualTestOnAllBuildersWithResultsLinks', 1, function() { 406 resetGlobals(); 407 loadBuildersList('@ToT - chromium.org', 'layout-tests'); 408 409 var builderName = 'WebKit Linux'; 392 var historyInstance = resetGlobals(); 393 historyInstance.crossDashboardState.group = '@ToT - webkit.org'; 394 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 395 396 var builderName = 'Apple Lion Release WK2 (Tests)'; 410 397 var test = 'dummytest.html'; 411 398 g_testToResultsMap[test] = [createResultsObjectForTest(test, builderName)]; … … 424 411 '<div>The following builders either don\'t run this test (e.g. it\'s skipped) or all runs passed:</div>' + 425 412 '<div class=skipped-builder-list>' + 426 '<div class=skipped-builder>WebKit Linux (dbg)</div><div class=skipped-builder>WebKit Mac10.7</div><div class=skipped-builder>WebKit Win</div><div class=skipped-builder>WebKit Win (dbg)</div>' + 427 '</div>' + 428 '<div class=expectations test=dummytest.html>' + 429 '<div><span class=link onclick="g_history.setQueryParameter(\'showExpectations\', true)">Show results</span> | ' + 430 '<span class=link onclick="g_history.setQueryParameter(\'showLargeExpectations\', true)">Show large thumbnails</span> | ' + 431 '<b>Only shows actual results/diffs from the most recent *failure* on each bot.</b></div>' + 432 '</div>'); 433 }); 434 435 test('htmlForIndividualTestOnAllBuildersWithResultsLinksWebkitMaster', 1, function() { 436 var historyInstance = resetGlobals(); 437 historyInstance.crossDashboardState.group = '@ToT - webkit.org'; 438 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 439 440 var builderName = 'Apple SnowLeopard Tests'; 441 var test = 'dummytest.html'; 442 g_testToResultsMap[test] = [createResultsObjectForTest(test, builderName)]; 443 444 equal(htmlForIndividualTestOnAllBuildersWithResultsLinks(test), 445 '<table class=test-table><thead><tr>' + 446 '<th sortValue=test><div class=table-header-content><span></span><span class=header-text>test</span></div></th>' + 447 '<th sortValue=bugs><div class=table-header-content><span></span><span class=header-text>bugs</span></div></th>' + 448 '<th sortValue=modifiers><div class=table-header-content><span></span><span class=header-text>modifiers</span></div></th>' + 449 '<th sortValue=expectations><div class=table-header-content><span></span><span class=header-text>expectations</span></div></th>' + 450 '<th sortValue=slowest><div class=table-header-content><span></span><span class=header-text>slowest run</span></div></th>' + 451 '<th sortValue=flakiness colspan=10000><div class=table-header-content><span></span><span class=header-text>flakiness (numbers are runtimes in seconds)</span></div></th>' + 452 '</tr></thead>' + 453 '<tbody></tbody>' + 454 '</table>' + 455 '<div>The following builders either don\'t run this test (e.g. it\'s skipped) or all runs passed:</div>' + 456 '<div class=skipped-builder-list>' + 457 '<div class=skipped-builder>Qt Linux Tests</div><div class=skipped-builder>Chromium Mac10.7 Tests</div><div class=skipped-builder>GTK Win</div>' + 413 '<div class=skipped-builder>Apple Lion Debug WK2 (Tests)</div><div class=skipped-builder>GTK Linux 64-bit Release</div><div class=skipped-builder>Qt Linux Tests</div>' + 458 414 '</div>' + 459 415 '<div class=expectations test=dummytest.html>' + … … 468 424 test('htmlForIndividualTests', 4, function() { 469 425 var historyInstance = resetGlobals(); 470 loadBuildersList('@ToT - chromium.org', 'layout-tests');426 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 471 427 var test1 = 'foo/nonexistant.html'; 472 428 var test2 = 'bar/nonexistant.html'; … … 564 520 }); 565 521 566 test('builderGroupIsToTWebKitAttribute', 2, function() {567 var dummyMaster = new builders.BuilderMaster('Chromium', 'dummyurl', {'layout-tests': {'builders': ['WebKit Linux', 'WebKit Linux (dbg)', 'WebKit Mac10.7', 'WebKit Win']}});568 var testBuilderGroups = {569 '@ToT - dummy.org': new BuilderGroup(BuilderGroup.TOT_WEBKIT),570 '@DEPS - dummy.org': new BuilderGroup(BuilderGroup.DEPS_WEBKIT),571 }572 573 var testJSONData = "{ \"Dummy Builder 1\": null, \"Dummy Builder 2\": null }";574 requestBuilderList(testBuilderGroups, 'ChromiumWebkit', '@ToT - dummy.org', testBuilderGroups['@ToT - dummy.org'], 'layout-tests');575 equal(testBuilderGroups['@ToT - dummy.org'].isToTWebKit, true);576 requestBuilderList(testBuilderGroups, 'ChromiumWebkit', '@DEPS - dummy.org', testBuilderGroups['@DEPS - dummy.org'], 'layout-tests');577 equal(testBuilderGroups['@DEPS - dummy.org'].isToTWebKit, false);578 });579 580 522 test('requestBuilderListAddsBuilderGroupEntry', 1, function() { 581 523 var testBuilderGroups = { '@ToT - dummy.org': null }; 582 524 var builderGroup = new BuilderGroup(BuilderGroup.TOT_WEBKIT); 583 525 var groupName = '@ToT - dummy.org'; 584 requestBuilderList(testBuilderGroups, ' ChromiumWebkit', groupName, builderGroup, 'layout-tests');526 requestBuilderList(testBuilderGroups, 'webkit.org', groupName, builderGroup, 'layout-tests'); 585 527 586 528 equal(testBuilderGroups['@ToT - dummy.org'], builderGroup); … … 691 633 test('changeTestTypeInvalidatesGroup', 1, function() { 692 634 var historyInstance = resetGlobals(); 693 var originalGroup = '@ToT - chromium.org';635 var originalGroup = '@ToT - webkit.org'; 694 636 var originalTestType = 'layout-tests'; 695 637 loadBuildersList(originalGroup, originalTestType); -
trunk/Tools/TestResultServer/static-dashboards/loader_unittests.js
r147457 r148349 45 45 } 46 46 47 try { 48 resourceLoader._loadingSteps = [loadingStep1, loadingStep2]; 49 resourceLoader.load(); 50 } 47 resourceLoader._loadingSteps = [loadingStep1, loadingStep2]; 48 resourceLoader.load(); 51 49 }); 52 50 53 51 // Total number of assertions is 1 for the deepEqual of the builder lists 54 52 // and then 2 per builder (one for ok, one for deepEqual of tests). 55 test('results files loading', 11, function() {53 test('results files loading', 9, function() { 56 54 resetGlobals(); 57 var expectedLoadedBuilders = [' WebKit Linux', 'WebKit Linux (dbg)', 'WebKit Mac10.7', 'WebKit Win', 'WebKit Win (dbg)'];55 var expectedLoadedBuilders = ['Apple Lion Debug WK2 (Tests)', 'Apple Lion Release WK2 (Tests)', 'GTK Linux 64-bit Release', 'Qt Linux Tests']; 58 56 var loadedBuilders = []; 59 57 var resourceLoader = new loader.Loader(); … … 68 66 var requestFunction = loader.request; 69 67 loader.request = function(url, successCallback, errorCallback) { 70 var builderName = /builder=([\w (). ]+)&/.exec(url)[1];68 var builderName = /builder=([\w ().-]+)&/.exec(url)[1]; 71 69 loadedBuilders.push(builderName); 72 70 successCallback({responseText: '{"version": 4, "' + builderName + '": {"secondsSinceEpoch": [' + Date.now() + '], "tests": {}}}'}); 73 71 } 74 72 75 loadBuildersList('@ToT - chromium.org', 'layout-tests');73 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 76 74 77 75 try { … … 85 83 resetGlobals(); 86 84 g_history.parseCrossDashboardParameters(); 87 var expectedLoadedPlatforms = [" chromium", "chromium-android", "efl", "efl-wk1", "efl-wk2", "gtk",88 "gtk-wk2","mac", "mac-lion", "mac-snowleopard", "qt", "win", "wk2"];85 var expectedLoadedPlatforms = ["efl", "efl-wk1", "efl-wk2", "gtk", "gtk-wk2", 86 "mac", "mac-lion", "mac-snowleopard", "qt", "win", "wk2"]; 89 87 var loadedPlatforms = []; 90 88 var resourceLoader = new loader.Loader(); … … 108 106 test('results file failing to load', 2, function() { 109 107 resetGlobals(); 110 loadBuildersList('@ToT - chromium.org', 'layout-tests');108 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 111 109 112 110 var resourceLoader = new loader.Loader(); … … 131 129 test('Default builder gets set.', 3, function() { 132 130 resetGlobals(); 133 loadBuildersList('@ToT - chromium.org', 'layout-tests');131 loadBuildersList('@ToT - webkit.org', 'layout-tests'); 134 132 135 133 var defaultBuilder = currentBuilderGroup().defaultBuilder();
Note: See TracChangeset
for help on using the changeset viewer.