Changeset 91212 in webkit


Ignore:
Timestamp:
Jul 18, 2011 2:51:18 PM (13 years ago)
Author:
ojan@chromium.org
Message:

update the flakiness dashboard to understand the new platforms/formats in test_expectations
​https://bugs.webkit.org/show_bug.cgi?id=64743

Reviewed by Adam Barth.

The test expectations format changed a long time ago and the flakiness dashboard
was never updated to match.

  • TestResultServer/static-dashboards/dashboard_base.js:
  • TestResultServer/static-dashboards/flakiness_dashboard.html:
  • TestResultServer/static-dashboards/flakiness_dashboard_tests.js:
  • TestResultServer/static-dashboards/timeline_explorer.html:
Location:
trunk/Tools
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r91210 r91212  
     12011-07-18  Ojan Vafai  <ojan@chromium.org>
     2
     3        update the flakiness dashboard to understand the new platforms/formats in test_expectations
     4        https://bugs.webkit.org/show_bug.cgi?id=64743
     5
     6        Reviewed by Adam Barth.
     7
     8        The test expectations format changed a long time ago and the flakiness dashboard
     9        was never updated to match.
     10
     11        * TestResultServer/static-dashboards/dashboard_base.js:
     12        * TestResultServer/static-dashboards/flakiness_dashboard.html:
     13        * TestResultServer/static-dashboards/flakiness_dashboard_tests.js:
     14        * TestResultServer/static-dashboards/timeline_explorer.html:
     15
    1162011-07-18  Eric Seidel  <eric@webkit.org>
    217

  • trunk/Tools/TestResultServer/static-dashboards/dashboard_base.js

    r91054 r91212  
    905905// @return Object with these properties:
    906906//     - testNames: array mapping test index to test names.
    907 //     - resultsByBuild: array of builds, for each build a (sparse) array of test
    908 //                                         results by test index.
    909 //     - flakyTests: array with the boolean value true at test indices that are
    910 //                                 considered flaky (more than one single-build failure).
    911 //     - flakyDeltasByBuild: array of builds, for each build a count of flaky
    912 //                                                 test results by expectation, as well as a total.
    913 function testResultsByBuild(builderResults)
     907//     - resultsByBuild: array of builds, for each build a (sparse) array of test results by test index.
     908//     - flakyTests: array with the boolean value true at test indices that are considered flaky (more than one single-build failure).
     909//     - flakyDeltasByBuild: array of builds, for each build a count of flaky test results by expectation, as well as a total.
     910function decompressResults(builderResults)
    914911{
    915912    var builderTestResults = builderResults[TESTS_KEY];
    916913    var buildCount = builderResults[FIXABLE_COUNTS_KEY].length;
    917     var testResultsByBuild = new Array(buildCount);
     914    var resultsByBuild = new Array(buildCount);
    918915    var flakyDeltasByBuild = new Array(buildCount);
    919916
    … …  
    923920        testCount++;
    924921    for (var i = 0; i < buildCount; i++) {
    925         testResultsByBuild[i] = new Array(testCount);
    926         testResultsByBuild[i][testCount - 1] = undefined;
     922        resultsByBuild[i] = new Array(testCount);
     923        resultsByBuild[i][testCount - 1] = undefined;
    927924        flakyDeltasByBuild[i] = {};
    928925    }
    … …  
    949946
    950947            for (var j = 0; j < count; j++) {
    951                 testResultsByBuild[currentBuildIndex++][testIndex] = value;
     948                resultsByBuild[currentBuildIndex++][testIndex] = value;
    952949                if (currentBuildIndex == buildCount)
    953950                    break;
    … …  
    992989    return {
    993990        testNames: testNames,
    994         resultsByBuild: testResultsByBuild,
     991        resultsByBuild: resultsByBuild,
    995992        flakyTests: flakyTests,
    996993        flakyDeltasByBuild: flakyDeltasByBuild

  • trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.html

    r91054 r91212  
    320320var TEST_URL_BASE_PATH = "http://svn.webkit.org/repository/webkit/trunk/LayoutTests/";
    321321var TEST_RESULTS_BASE_PATH = 'http://build.chromium.org/f/chromium/layout_test_results/';
     322
     323// FIXME: Make this an array.
    322324var PLATFORMS = {
    323 'MAC': 'MAC',
    324 'LINUX': 'LINUX',
    325 'WIN': 'WIN',
    326 'WIN-XP': 'WIN-XP',
    327 'WIN-VISTA': 'WIN-VISTA'
     325    'SNOWLEOPARD': 'SNOWLEOPARD',
     326    'LEOPARD': 'LEOPARD',
     327    'XP': 'XP',
     328    'VISTA': 'VISTA',
     329    'WIN7': 'WIN7',
     330    'LUCID': 'LUCID'
    328331};
     332
     333// FIXME: Make the g_allExpectations data structure explicitly list every platform instead of having a fallbacks concept.
    329334var PLATFORM_FALLBACKS = {
    330 'WIN': 'ALL',
    331 'WIN-XP': 'WIN',
    332 'WIN-VISTA': 'WIN',
    333 'LINUX': 'ALL',
    334 'MAC': 'ALL'
     335    'WIN': 'ALL',
     336    'XP': 'WIN',
     337    'VISTA': 'WIN',
     338    'MAC': 'ALL',
     339    'SNOWLEOPARD': 'MAC',
     340    'LEOPARD': 'MAC',
     341    'LINUX': 'ALL',
     342    'LUCID': 'LINUX'
    335343};
    336344var BUILD_TYPES = {'DEBUG': 'DBG', 'RELEASE': 'RELEASE'};
    337345var MIN_SECONDS_FOR_SLOW_TEST = 4;
    338346var MIN_SECONDS_FOR_SLOW_TEST_DEBUG = 2 * MIN_SECONDS_FOR_SLOW_TEST;
    339 var FAIL_RESULTS = ['IMAGE', 'IMAGE+TEXT', 'TEXT', 'SIMPLIFIED', 'MISSING'];
     347var FAIL_RESULTS = ['IMAGE', 'IMAGE+TEXT', 'TEXT', 'MISSING'];
    340348var CHUNK_SIZE = 25;
    341349var MAX_RESULTS = 1500;
    … …  
    526534        // or build type, assume Windows Release.
    527535        var currentBuildUppercase = builderName.toUpperCase();
    528         var platform = matchingElement(currentBuildUppercase, PLATFORMS) || 'WIN';
    529         var buildType = matchingElement(currentBuildUppercase, BUILD_TYPES) || 'RELEASE';
     536       
     537        var platform;
     538        if (stringContains(currentBuildUppercase, 'MAC')) {
     539            if (stringContains(currentBuildUppercase, '10.5'))
     540                platform = 'LEOPARD';
     541            else
     542                platform = 'SNOWLEOPARD';
     543        } else if (stringContains(currentBuildUppercase, 'WIN7'))
     544            platform = 'WIN7';
     545        else if (stringContains(currentBuildUppercase, 'VISTA'))
     546            platform = 'VISTA';
     547        else if (stringContains(currentBuildUppercase, 'WIN'))
     548            platform = 'XP';
     549        else if (stringContains(currentBuildUppercase, 'LINUX'))
     550            platform = 'LUCID';
     551        else {
     552            platform = '';
     553            console.log('Could not resolve platform for builder: ' + builderName);
     554        }
     555       
     556        var buildType = stringContains(currentBuildUppercase, 'DBG') ? 'DEBUG' : 'RELEASE';
    530557        g_perBuilderPlatformAndBuildType[builderName] = {platform: platform, buildType: buildType};
    531558    }
    … …  
    734761    var platformObject;
    735762    while (platform && !(platformObject = testObject[platform]))
    736       platform = PLATFORM_FALLBACKS[platform];
     763        platform = PLATFORM_FALLBACKS[platform];
    737764
    738765    if (platformObject) {
    … …  
    837864            addFallbacks(function(buildTypeKey) {
    838865                // Setting the ALL key overrides any previously seen expectations.
    839                 if (platformKey == ALL) {
    840                     // FIXME: This is broken. PLATFORMS is an object, not an array.
    841                     // Figure out if we should just be deleting this or fixing it.
    842                     for (var i = 0; i < PLATFORMS.length; i++) {
    843                       if (PLATFORMS[i] in g_allExpectations[test]) {
    844                         // Setting the ALL key overrides any previously seen
    845                         // expectations.
    846                         if (buildTypeKey == ALL)
    847                           g_allExpectations[test][PLATFORMS[i]] = {};
    848                       }
     866                if (platformKey == ALL && g_allExpectations[test]) {
     867                    for (var platform in PLATFORM_FALLBACKS) {
     868                          if (platform in g_allExpectations[test]) {
     869                              // Setting the ALL key overrides any previously seen
     870                              // expectations.
     871                              if (buildTypeKey == ALL)
     872                                  g_allExpectations[test][platform] = {};
     873                          }
    849874                    }
    850875                } else if (buildTypeKey == ALL && g_allExpectations[test])
    … …  
    860885                testHolder[platformKey][buildTypeKey] = expectations[j];
    861886            }, modifiers, BUILD_TYPES);
    862         }, modifiers, PLATFORMS);
     887        }, modifiers, PLATFORM_FALLBACKS);
    863888    }
    864889}
    … …  
    9871012        resultsForTest.rawResults = rawResults;
    9881013
    989         // FIXME: Switch to testResultsByBuild
     1014        // FIXME: Switch to resultsByBuild
    9901015        var times = resultsForTest.rawTimes;
    9911016        var numTimesSeen = 0;
    … …  
    16901715        for (var i = 0; i < results.length; i++) {
    16911716            var thisResult = results[i];
     1717           
    16921718            if (!thisResult.missing && !thisResult.extra)
    16931719                continue;

  • trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_tests.js

    r91054 r91212  
    3434// arguments.
    3535
    36 // FIXME: Add tests for processMissingAndExtraExpectations
     36// FIXME: move this over to using qunit
    3737
    3838// Clears out the global objects modified or used by processExpectations and
    … …  
    4646    g_resultsByBuilder = {};
    4747    g_builders = {};
     48    g_allExpectations = null;
     49    g_allTests = null;
    4850}
    4951
    … …  
    7173    populateExpectationsData(resultsForTest);
    7274
    73     assertEquals(resultsForTest, resultsForTest.expectations, expectations);
    74     assertEquals(resultsForTest, resultsForTest.modifiers, modifiers);
    75 }
    76 
    77 function assertEquals(resultsForTest, actual, expected)
    78 {
    79     if (expected !== actual)
    80         throw Error('Builder: ' + resultsForTest.builder + ' test: ' + resultsForTest.test + ' got: ' + actual + ' expected: ' + expected);
     75    var message = 'Builder: ' + resultsForTest.builder + ' test: ' + resultsForTest.test;
     76    assertEquals(resultsForTest.expectations, expectations, message);
     77    assertEquals(resultsForTest.modifiers, modifiers, message);
     78}
     79
     80function assertEquals(actual, expected, message)
     81{
     82    if (expected !== actual) {
     83        if (message)
     84            message += ' ';
     85        else
     86            message = '';
     87
     88        throw Error(message + 'got: ' + actual + ' expected: ' + expected);
     89    }
    8190}
    8291
    … …  
    8594function testReleaseFail()
    8695{
    87     var builder = 'Webkit';
     96    var builder = 'Webkit Win';
    8897    var test = 'foo/1.html';
    8998    var expectationsArray = [
    … …  
    96105function testReleaseFailDebugCrashReleaseBuilder()
    97106{
    98     var builder = 'Webkit';
     107    var builder = 'Webkit Win';
    99108    var test = 'foo/1.html';
    100109    var expectationsArray = [
    … …  
    108117function testReleaseFailDebugCrashDebugBuilder()
    109118{
    110     var builder = 'Webkit(dbg)';
     119    var builder = 'Webkit Win (dbg)';
    111120    var test = 'foo/1.html';
    112121    var expectationsArray = [
    … …  
    128137        {'modifiers': 'LINUX DEBUG', 'expectations': 'CRASH'},
    129138    ];
    130     runExpectationsTest('Webkit', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
    131     runExpectationsTest('Webkit (dbg)(3)', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
     139    runExpectationsTest('Webkit Win', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
     140    runExpectationsTest('Webkit Win (dbg)(3)', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
    132141    runExpectationsTest('Webkit Linux', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
    133142    runExpectationsTest('Webkit Linux (dbg)(3)', test, 'CRASH', 'LINUX DEBUG');
    134143    runExpectationsTest('Webkit Mac10.5', test, 'FAIL', 'WONTFIX MAC');
    135144    runExpectationsTest('Webkit Mac10.5 (dbg)(3)', test, 'FAIL', 'WONTFIX MAC');
     145}
     146
     147function testPlatformAndBuildType()
     148{
     149    var runPlatformAndBuildTypeTest = function(builder, expectedPlatform, expectedBuildType) {
     150        g_perBuilderPlatformAndBuildType = {};
     151        buildInfo = platformAndBuildType(builder);
     152        var message = 'Builder: ' + builder;
     153        assertEquals(buildInfo.platform, expectedPlatform, message);
     154        assertEquals(buildInfo.buildType, expectedBuildType, message);
     155    }
     156    runPlatformAndBuildTypeTest('Webkit Win (deps)', 'XP', 'RELEASE');
     157    runPlatformAndBuildTypeTest('Webkit Win (deps)(dbg)(1)', 'XP', 'DEBUG');
     158    runPlatformAndBuildTypeTest('Webkit Win (deps)(dbg)(2)', 'XP', 'DEBUG');
     159    runPlatformAndBuildTypeTest('Webkit Linux (deps)', 'LUCID', 'RELEASE');
     160    runPlatformAndBuildTypeTest('Webkit Linux (deps)(dbg)(1)', 'LUCID', 'DEBUG');
     161    runPlatformAndBuildTypeTest('Webkit Linux (deps)(dbg)(2)', 'LUCID', 'DEBUG');
     162    runPlatformAndBuildTypeTest('Webkit Mac10.6 (deps)', 'SNOWLEOPARD', 'RELEASE');
     163    runPlatformAndBuildTypeTest('Webkit Mac10.6 (deps)(dbg)(1)', 'SNOWLEOPARD', 'DEBUG');
     164    runPlatformAndBuildTypeTest('Webkit Mac10.6 (deps)(dbg)(2)', 'SNOWLEOPARD', 'DEBUG');
     165    runPlatformAndBuildTypeTest('Webkit Win', 'XP', 'RELEASE');
     166    runPlatformAndBuildTypeTest('Webkit Vista', 'VISTA', 'RELEASE');
     167    runPlatformAndBuildTypeTest('Webkit Win7', 'WIN7', 'RELEASE');
     168    runPlatformAndBuildTypeTest('Webkit Win (dbg)(1)', 'XP', 'DEBUG');
     169    runPlatformAndBuildTypeTest('Webkit Win (dbg)(2)', 'XP', 'DEBUG');
     170    runPlatformAndBuildTypeTest('Webkit Linux', 'LUCID', 'RELEASE');
     171    runPlatformAndBuildTypeTest('Webkit Linux 32', 'LUCID', 'RELEASE');
     172    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(1)', 'LUCID', 'DEBUG');
     173    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(2)', 'LUCID', 'DEBUG');
     174    runPlatformAndBuildTypeTest('Webkit Mac10.5', 'LEOPARD', 'RELEASE');
     175    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(1)', 'LEOPARD', 'DEBUG');
     176    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(2)', 'LEOPARD', 'DEBUG');
     177    runPlatformAndBuildTypeTest('Webkit Mac10.6', 'SNOWLEOPARD', 'RELEASE');
     178    runPlatformAndBuildTypeTest('Webkit Mac10.6 (dbg)', 'SNOWLEOPARD', 'DEBUG');
     179    runPlatformAndBuildTypeTest('Chromium Win Release (Tests)', 'XP', 'RELEASE');
     180    runPlatformAndBuildTypeTest('Chromium Linux Release (Tests)', 'LUCID', 'RELEASE');
     181    runPlatformAndBuildTypeTest('Chromium Mac Release (Tests)', 'SNOWLEOPARD', 'RELEASE');
     182    runPlatformAndBuildTypeTest('Webkit Win - GPU', 'XP', 'RELEASE');
     183    runPlatformAndBuildTypeTest('Webkit Vista - GPU', 'VISTA', 'RELEASE');
     184    runPlatformAndBuildTypeTest('Webkit Win7 - GPU', 'WIN7', 'RELEASE');
     185    runPlatformAndBuildTypeTest('Webkit Win (dbg)(1) - GPU', 'XP', 'DEBUG');
     186    runPlatformAndBuildTypeTest('Webkit Win (dbg)(2) - GPU', 'XP', 'DEBUG');
     187    runPlatformAndBuildTypeTest('Webkit Linux - GPU', 'LUCID', 'RELEASE');
     188    runPlatformAndBuildTypeTest('Webkit Linux 32 - GPU', 'LUCID', 'RELEASE');
     189    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(1) - GPU', 'LUCID', 'DEBUG');
     190    runPlatformAndBuildTypeTest('Webkit Linux (dbg)(2) - GPU', 'LUCID', 'DEBUG');
     191    runPlatformAndBuildTypeTest('Webkit Mac10.5 - GPU', 'LEOPARD', 'RELEASE');
     192    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(1) - GPU', 'LEOPARD', 'DEBUG');
     193    runPlatformAndBuildTypeTest('Webkit Mac10.5 (dbg)(2) - GPU', 'LEOPARD', 'DEBUG');
     194    runPlatformAndBuildTypeTest('Webkit Mac10.6 - GPU', 'SNOWLEOPARD', 'RELEASE');
     195    runPlatformAndBuildTypeTest('Webkit Mac10.6 (dbg) - GPU', 'SNOWLEOPARD', 'DEBUG');
     196    runPlatformAndBuildTypeTest('GPU Win7 Tests - GPU', 'WIN7', 'RELEASE');
     197    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(1) - GPU', 'WIN7', 'DEBUG');
     198    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(2) - GPU', 'WIN7', 'DEBUG');
     199    runPlatformAndBuildTypeTest('GPU Win7 x64 Tests (dbg)(1) - GPU', 'WIN7', 'DEBUG');
     200    runPlatformAndBuildTypeTest('GPU Win7 x64 Tests (dbg)(2) - GPU', 'WIN7', 'DEBUG');
     201    runPlatformAndBuildTypeTest('GPU Vista Tests (dbg)(1) - GPU', 'VISTA', 'DEBUG');
     202    runPlatformAndBuildTypeTest('GPU Vista Tests (dbg)(2) - GPU', 'VISTA', 'DEBUG');
     203    runPlatformAndBuildTypeTest('GPU Vista x64 Tests (dbg) - GPU', 'VISTA', 'DEBUG');
     204    runPlatformAndBuildTypeTest('GPU Mac 10.6 Tests - GPU', 'SNOWLEOPARD', 'RELEASE');
     205    runPlatformAndBuildTypeTest('GPU Mac 10.6 Tests (dbg) - GPU', 'SNOWLEOPARD', 'DEBUG');
     206    runPlatformAndBuildTypeTest('GPU Mac 10.5 Tests (dbg) - GPU', 'LEOPARD', 'DEBUG');
     207    runPlatformAndBuildTypeTest('GPU Linux Tests (dbg)(1) - GPU', 'LUCID', 'DEBUG');
     208    runPlatformAndBuildTypeTest('GPU Linux Tests (dbg)(2) - GPU', 'LUCID', 'DEBUG');
     209    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(1) - GPU', 'WIN7', 'DEBUG');
     210    runPlatformAndBuildTypeTest('GPU Win7 Tests (dbg)(2) - GPU', 'WIN7', 'DEBUG');
     211    runPlatformAndBuildTypeTest('GPU Linux Tests x64 - GPU', 'LUCID', 'RELEASE');
    136212}
    137213

  • trunk/Tools/TestResultServer/static-dashboards/timeline_explorer.html

    r91054 r91212  
    146146{
    147147    var startTime = Date.now();
    148     g_currentBuilderTestResults = testResultsByBuild(g_resultsByBuilder[g_currentState.builder]);
     148    g_currentBuilderTestResults = decompressResults(g_resultsByBuilder[g_currentState.builder]);
    149149    console.log( 'Time to get test results by build: ' + (Date.now() - startTime));
    150150}
Note: See TracChangeset for help on using the changeset viewer.