Changeset 129079 in webkit


Ignore:
Timestamp:
Sep 19, 2012 5:59:24 PM (12 years ago)
Author:
dpranke@chromium.org
Message:

update flakiness dashboard after cutover to new test expectations syntax
​https://bugs.webkit.org/show_bug.cgi?id=97152

Reviewed by Ryosuke Niwa.

This change clones the TestExpectation parsing state machine
from python into javascript.

  • TestResultServer/static-dashboards/flakiness_dashboard.js:

(parsedExpectations.lines.forEach):
(parsedExpectations):

  • TestResultServer/static-dashboards/flakiness_dashboard_unittests.js:
Location:
trunk/Tools
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r129073 r129079  
     12012-09-19  Dirk Pranke  <dpranke@chromium.org>
     2
     3        update flakiness dashboard after cutover to new test expectations syntax
     4        https://bugs.webkit.org/show_bug.cgi?id=97152
     5
     6        Reviewed by Ryosuke Niwa.
     7
     8        This change clones the TestExpectation parsing state machine
     9        from python into javascript.
     10
     11        * TestResultServer/static-dashboards/flakiness_dashboard.js:
     12        (parsedExpectations.lines.forEach):
     13        (parsedExpectations):
     14        * TestResultServer/static-dashboards/flakiness_dashboard_unittests.js:
     15
    1162012-09-19  Dana Jansens  <danakj@chromium.org>
    217

  • trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard.js

    r128321 r129079  
    589589    lines.forEach(function(line) {
    590590        line = trimString(line);
    591         if (!line || startsWith(line, '//'))
     591        if (!line || startsWith(line, '#'))
    592592            return;
    593593
    594         // FIXME: Make this robust against not having modifiers and/or expectations.
    595         // Right now, run-webkit-tests doesn't allow such lines, but it may in the future.
    596         var match = line.match(/([^:]+)*:([^=]+)=(.*)/);
    597         if (!match) {
    598             console.error('Line could not be parsed: ' + line);
    599             return;
     594        // This code mimics _tokenize_line_using_new_format() in
     595        // Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
     596        //
     597        // FIXME: consider doing more error checking here.
     598        //
     599        // FIXME: Clean this all up once we've fully cut over to the new syntax.
     600        var tokens = line.split(/\s+/)
     601        var parsed_bugs = [];
     602        var parsed_modifiers = [];
     603        var parsed_path;
     604        var parsed_expectations = [];
     605        var state = 'start';
     606
     607        // This clones _configuration_tokens_list in test_expectations.py.
     608        // FIXME: unify with the platforms constants at the top of the file.
     609        var configuration_tokens = {
     610            'Release': 'RELEASE',
     611            'Debug': 'DEBUG',
     612            'Mac': 'MAC',
     613            'Win': 'Win',
     614            'Linux': 'LINUX',
     615            'SnowLeopard': 'SNOWLEOPARD',
     616            'Lion': 'LION',
     617            'MountainLion': 'MOUNTAINLION',
     618            'Win7': 'WIN7',
     619            'XP': 'XP',
     620            'Vista': 'VISTA',
     621            'Android': 'ANDROID',
     622        };
     623
     624        var expectation_tokens = {
     625            'Crash': 'CRASH',
     626            'Failure': 'FAIL',
     627            'ImageOnlyFailure': 'IMAGE',
     628            'Missing': 'MISSING',
     629            'Pass': 'PASS',
     630            'Rebaseline': 'REBASELINE',
     631            'Skip': 'SKIP',
     632            'Slow': 'SLOW',
     633            'Timeout': 'TIMEOUT',
     634            'WontFix': 'WONTFIX',
     635        };
     636
     637           
     638        tokens.forEach(function(token) {
     639          if (token.indexOf('Bug') != -1 ||
     640              token.indexOf('webkit.org') != -1 ||
     641              token.indexOf('crbug.com') != -1 ||
     642              token.indexOf('code.google.com') != -1) {
     643              parsed_bugs.push(token);
     644          } else if (token == '[') {
     645              if (state == 'start') {
     646                  state = 'configuration';
     647              } else if (state == 'name_found') {
     648                  state = 'expectations';
     649              }
     650          } else if (token == ']') {
     651              if (state == 'configuration') {
     652                  state = 'name';
     653              } else if (state == 'expectations') {
     654                  state = 'done';
     655              }
     656          } else if (state == 'configuration') {
     657              parsed_modifiers.push(configuration_tokens[token]);
     658          } else if (state == 'expectations') {
     659              if (token == 'Rebaseline' || token == 'Skip' || token == 'Slow' || token == 'WontFix') {
     660                  parsed_modifiers.push(token.toUpperCase());
     661              } else {
     662                  parsed_expectations.push(expectation_tokens[token]);
     663              }
     664          } else if (token == '#') {
     665              state = 'done';
     666          } else if (state == 'name' || state == 'start') {
     667              parsed_path = token;
     668              state = 'name_found';
     669          }
     670        });
     671
     672        if (!parsed_expectations.length) {
     673            if (parsed_modifiers.indexOf('Slow') == -1) {
     674                parsed_modifiers.push('Skip');
     675                parsed_expectations = ['Pass'];
     676            }
    600677        }
    601678
    602679        // FIXME: Should we include line number and comment lines here?
    603680        expectations.push({
    604             modifiers: trimString(match[1]),
    605             path: trimString(match[2]),
    606             expectations: trimString(match[3])
     681            modifiers: parsed_bugs.concat(parsed_modifiers).join(' '),
     682            path: parsed_path,
     683            expectations: parsed_expectations.join(' '),
    607684        });
    608685    });

  • trunk/Tools/TestResultServer/static-dashboards/flakiness_dashboard_unittests.js

    r128321 r129079  
    8585        {'modifiers': 'RELEASE', 'expectations': 'FAIL'}
    8686    ];
    87     g_expectations = 'RELEASE : ' + test + ' = FAIL';
     87    g_expectations = '[ Release ] ' + test + ' [ Failure ]';
    8888    runExpectationsTest(builder, test, 'FAIL', 'RELEASE');
    8989});
    … …  
    9797        {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
    9898    ];
    99     g_expectations = 'RELEASE : ' + test + ' = FAIL\n' +
    100         'DEBUG : ' + test + ' = CRASH';
     99    g_expectations = '[ Release ] ' + test + ' [ Failure ]\n' +
     100        '[ Debug ] ' + test + ' [ Crash ]';
    101101    runExpectationsTest(builder, test, 'FAIL', 'RELEASE');
    102102});
    … …  
    110110        {'modifiers': 'DEBUG', 'expectations': 'CRASH'}
    111111    ];
    112     g_expectations = 'RELEASE : ' + test + ' = FAIL\n' +
    113         'DEBUG : ' + test + ' = CRASH';
     112    g_expectations = '[ Release ] ' + test + ' [ Failure ]\n' +
     113        '[ Debug ] ' + test + ' [ Crash ]';
    114114    runExpectationsTest(builder, test, 'CRASH', 'DEBUG');
    115115});
    … …  
    118118    resetGlobals();
    119119    var test = 'bar/1.html';
    120     g_expectations = 'WONTFIX : bar = FAIL PASS TIMEOUT\n' +
    121         'WONTFIX MAC : ' + test + ' = FAIL\n' +
    122         'LINUX DEBUG : ' + test + ' = CRASH';
     120    g_expectations = 'bar [ WontFix Failure Pass Timeout ]\n' +
     121        '[ Mac ] ' + test + ' [ WontFix Failure ]\n' +
     122        '[ Linux Debug ] ' + test + ' [ Crash ]';
    123123   
    124124    runExpectationsTest('Webkit Win', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
    … …  
    126126    runExpectationsTest('Webkit Linux', test, 'FAIL PASS TIMEOUT', 'WONTFIX');
    127127    runExpectationsTest('Webkit Linux (dbg)(3)', test, 'CRASH', 'LINUX DEBUG');
    128     runExpectationsTest('Webkit Mac10.7', test, 'FAIL', 'WONTFIX MAC');
    129     runExpectationsTest('Webkit Mac10.7 (dbg)(3)', test, 'FAIL', 'WONTFIX MAC');
     128    runExpectationsTest('Webkit Mac10.7', test, 'FAIL', 'MAC WONTFIX');
     129    runExpectationsTest('Webkit Mac10.7 (dbg)(3)', test, 'FAIL', 'MAC WONTFIX');
    130130});
    131131
    … …  
    186186
    187187test('realModifiers', 3, function() {
    188     equal(realModifiers('BUGFOO LINUX LION WIN DEBUG SLOW'), 'SLOW');
    189     equal(realModifiers('BUGFOO LUCID MAC XP RELEASE SKIP'), 'SKIP');
    190     equal(realModifiers('BUGFOO'), '');
     188    equal(realModifiers('BUG(Foo) LINUX LION WIN DEBUG SLOW'), 'SLOW');
     189    equal(realModifiers('BUG(Foo) LUCID MAC XP RELEASE SKIP'), 'SKIP');
     190    equal(realModifiers('BUG(Foo)'), '');
    191191});
    192192
    … …  
    220220    }
    221221
    222     g_expectations = 'BUG123 : foo = FAIL PASS CRASH\n' +
    223         'RELEASE BUGFOO : foo/test1.html = FAIL\n' +
    224         'DEBUG : foo/test1.html = CRASH\n' +
    225         'BUG456 : foo/test2.html = FAIL\n' +
    226         'LINUX DEBUG : foo/test2.html = CRASH\n' +
    227         'RELEASE : test1.html = FAIL\n' +
    228         'DEBUG : test1.html = CRASH\n' +
    229         'WIN7 : http/tests/appcache/interrupted-update.html = TIMEOUT\n' +
    230         'MAC LINUX XP : http/tests/appcache/interrupted-update.html = FAIL\n';
     222    g_expectations = 'Bug(123) foo [ Failure Pass Crash ]\n' +
     223        'Bug(Foo) [ Release ] foo/test1.html [ Failure ]\n' +
     224        '[ Debug ] foo/test1.html [ Crash ]\n' +
     225        'Bug(456) foo/test2.html [ Failure ]\n' +
     226        '[ Linux Debug ] foo/test2.html [ Crash ]\n' +
     227        '[ Release ] test1.html [ Failure ]\n' +
     228        '[ Debug ] test1.html [ Crash ]\n' +
     229        '[ Win7 ] http/tests/appcache/interrupted-update.html [ Timeout ]\n' +
     230        '[ Mac Linux XP ] http/tests/appcache/interrupted-update.html [ Failure ]\n';
    231231
    232232    processExpectations();
    … …  
    236236
    237237    var expectations = getExpectations('foo/test1.html', 'LUCID', 'RELEASE');
    238     equal(JSON.stringify(expectations), '{"modifiers":"RELEASE BUGFOO","expectations":"FAIL"}');
     238    equal(JSON.stringify(expectations), '{"modifiers":"Bug(Foo) RELEASE","expectations":"FAIL"}');
    239239
    240240    var expectations = getExpectations('foo/test2.html', 'LUCID', 'RELEASE');
    241     equal(JSON.stringify(expectations), '{"modifiers":"BUG456","expectations":"FAIL"}');
     241    equal(JSON.stringify(expectations), '{"modifiers":"Bug(456)","expectations":"FAIL"}');
    242242
    243243    var expectations = getExpectations('foo/test2.html', 'LION', 'DEBUG');
    244     equal(JSON.stringify(expectations), '{"modifiers":"BUG456","expectations":"FAIL"}');
     244    equal(JSON.stringify(expectations), '{"modifiers":"Bug(456)","expectations":"FAIL"}');
    245245
    246246    var expectations = getExpectations('foo/test2.html', 'LUCID', 'DEBUG');
    … …  
    248248
    249249    var expectations = getExpectations('foo/test3.html', 'LUCID', 'DEBUG');
    250     equal(JSON.stringify(expectations), '{"modifiers":"BUG123","expectations":"FAIL PASS CRASH"}');
     250    equal(JSON.stringify(expectations), '{"modifiers":"Bug(123)","expectations":"FAIL PASS CRASH"}');
    251251
    252252    var expectations = getExpectations('test1.html', 'XP', 'DEBUG');
Note: See TracChangeset for help on using the changeset viewer.