Changeset 85798 in webkit


Ignore:
Timestamp:
May 4, 2011 3:50:35 PM (13 years ago)
Author:
ojan@chromium.org
Message:

2011-05-04 Ojan Vafai <ojan@chromium.org>

Reviewed by Eric Seidel.

[results.html] put flaky tests into their own table
https://bugs.webkit.org/show_bug.cgi?id=60214

  • fast/harness/resources/results-test.js:
  • fast/harness/results-expected.txt:
  • fast/harness/results.html:
Location:
trunk/LayoutTests
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/LayoutTests/ChangeLog

    r85796 r85798  
     12011-05-04  Ojan Vafai  <ojan@chromium.org>
     2
     3        Reviewed by Eric Seidel.
     4
     5        [results.html] put flaky tests into their own table
     6        https://bugs.webkit.org/show_bug.cgi?id=60214
     7
     8        * fast/harness/resources/results-test.js:
     9        * fast/harness/results-expected.txt:
     10        * fast/harness/results.html:
     11
    1122011-05-04  Ojan Vafai  <ojan@chromium.org>
    213
  • trunk/LayoutTests/fast/harness/resources/results-test.js

    r85796 r85798  
    270270    subtree['bar-flaky-fail.html'] = mockExpectation('PASS TEXT', 'IMAGE PASS');
    271271    runTest(results, function() {
    272         assertTrue(document.getElementById('results-table'));
     272        assertTrue(!document.getElementById('results-table'));
     273        assertTrue(document.getElementById('flaky-tests-table'));
    273274        assertTrue(document.body.textContent.indexOf('bar-flaky-fail.html') != -1);
     275    });
     276
     277    results = mockResults();
     278    var subtree = results.tests['foo'] = {}
     279    subtree['bar-flaky-expected.html'] = mockExpectation('PASS FAIL', 'PASS TEXT');
     280    runTest(results, function() {
     281        assertTrue(!document.getElementById('results-table'));
     282        assertTrue(document.getElementById('flaky-tests-table'));
     283        assertTrue(document.body.textContent.indexOf('bar-flaky-expected.html') != -1);
     284        assertTrue(document.querySelector('tbody').className == 'expected');
    274285    });
    275286
     
    290301    subtree['bar1.html'] = mockExpectation('CRASH', 'PASS');
    291302    subtree['bar2.html'] = mockExpectation('IMAGE', 'PASS');
     303    subtree['flaky-fail.html'] = mockExpectation('PASS TEXT', 'IMAGE PASS');
    292304    results.uses_expectations_file = false;
    293305    runTest(results, function() {
     
    295307        assertTrue(!document.querySelector('tbody.expected'));
    296308        assertTrue(!document.getElementById('passes-table'));
     309        assertTrue(document.querySelectorAll('#flaky-tests-table tbody td').length == 4);
    297310    });
    298311
  • trunk/LayoutTests/fast/harness/results-expected.txt

    r85796 r85798  
    9898TEST-16: PASS
    9999TEST-16: PASS
     100TEST-16: PASS
     101TEST-17: PASS
     102TEST-17: PASS
    100103TEST-17: PASS
    101104TEST-17: PASS
    102105TEST-18: PASS
    103106TEST-18: PASS
    104 TEST-18: PASS
     107TEST-19: PASS
     108TEST-19: PASS
    105109TEST-19: PASS
    106110TEST-19: PASS
     
    108112TEST-20: PASS
    109113TEST-21: PASS
     114TEST-21: PASS
    110115TEST-22: PASS
    111 TEST-22: PASS
     116TEST-23: PASS
    112117TEST-23: PASS
    113118TEST-24: PASS
    114119TEST-25: PASS
    115120TEST-26: PASS
    116 TEST-26: PASS
    117 TEST-26: PASS
    118121TEST-27: PASS
    119122TEST-27: PASS
     123TEST-27: PASS
     124TEST-28: PASS
     125TEST-28: PASS
  • trunk/LayoutTests/fast/harness/results.html

    r85796 r85798  
    153153        g_state = {
    154154            crashTests: [],
     155            flakyTests: [],
    155156            hasHttpTests: false,
    156157            hasImageFailures: false,
     
    159160            results: {},
    160161            shouldToggleImages: true,
     162            nonFlakyFailingTests: [],
    161163            testsWithStderr: [],
    162164            timeoutTests: [],
     
    366368}
    367369
    368 function testObjectFor(test)
    369 {
    370     var parts = test.split('/');
    371     var tree = globalState().results.tests;
    372     for (var i = 0; i < parts.length; i++) {
    373         if (parts[i] in tree)
    374             tree = tree[parts[i]];
    375         else {
    376             console.error('Test not in the results tree: ' + test);
    377             return null;
    378         }
    379     }
    380     return tree;
    381 }
    382 
    383370function processGlobalStateFor(testObject)
    384371{
    385372    var test = testObject.name;
    386373    if (testObject.has_stderr)
    387         globalState().testsWithStderr.push(test);
     374        globalState().testsWithStderr.push(testObject);
    388375
    389376    globalState().hasHttpTests = globalState().hasHttpTests || test.indexOf('http/') == 0;
    390377
    391     var actual = testObject.actual;
     378    var actual = testObject.actual;   
    392379    if (actual == 'MISSING') {
    393380        // FIXME: make sure that new-run-webkit-tests spits out an -actual.txt file for
    394381        // tests with MISSING results.
    395         globalState().newTests.push(test);
     382        globalState().newTests.push(testObject);
    396383        return;
    397384    }
     
    411398    }
    412399
    413     if (actual == 'PASS' && expected != 'PASS')
    414         globalState().unexpectedPassTests.push(test);           
    415 
    416     if (globalState().results.uses_expectations_file && testObject.isExpected)
    417         return;
    418 
    419     // FIXME: Show some indication in the UI for a test that doesn't crash on the first run but crashes on the second run.
    420     if (actual.indexOf('CRASH') != -1) {
    421         globalState().crashTests.push(test);
    422         return;
    423     }
    424 
    425     // FIXME: Show some indication in the UI for a test only times out on one run?
    426     if (actual.indexOf('TIMEOUT') != -1)
    427         globalState().timeoutTests.push(test);
     400    if (actual.indexOf(' ') != -1) {
     401        globalState().flakyTests.push(testObject);
     402        return;
     403    }
     404
     405    if (actual == 'PASS' && expected != 'PASS') {
     406        globalState().unexpectedPassTests.push(testObject);
     407        return;
     408    }
     409
     410    if (actual == 'CRASH' && expected != 'CRASH') {
     411        globalState().crashTests.push(testObject);
     412        return;
     413    }
     414
     415    if (actual == 'TIMEOUT' && expected != 'TIMEOUT') {
     416        globalState().timeoutTests.push(testObject);
     417        return;
     418    }
     419   
     420    globalState().nonFlakyFailingTests.push(testObject);
    428421}
    429422
     
    511504    row += '</td>';
    512505
     506    if (globalState().results.uses_expectations_file || actual.indexOf(' ') != -1)
     507        row += '<td>' + actual + '</td>';
     508
    513509    if (globalState().results.uses_expectations_file)
    514         row += '<td>' + actual + '</td><td>' + testObject.expected + '</td>';
     510        row += '<td>' + testObject.expected + '</td>';
    515511
    516512    row += '</tr></tbody>';
     
    534530}
    535531
    536 function tableRows()
    537 {
    538     var html = '';
    539     forEachTest(function(testObject) {
    540         processGlobalStateFor(testObject)
    541         var test = testObject.name;
    542         if (globalState().unexpectedPassTests.indexOf(test) == -1 &&
    543             globalState().newTests.indexOf(test) == -1 &&
    544             globalState().crashTests.indexOf(test) == -1 &&
    545             globalState().timeoutTests.indexOf(test) == -1)
    546             html += tableRow(testObject);
    547     });
    548     return html;
    549 }
    550 
    551532function testList(tests, header, tableId)
    552533{
     
    559540
    560541    for (var i = 0; i < tests.length; i++) {
    561         var test = tests[i];
     542        var testObject = tests[i];
     543        var test = testObject.name;
    562544        html += '<tbody><tr><td>';
    563545        html += (tableId == 'passes-table') ? testLink(test) : testLinkWithExpandButton(test);
     
    567549            html += resultLink(stripExtension(test), '-stderr.txt', 'stderr');
    568550        else if (tableId == 'passes-table')
    569             html += testObjectFor(test).expected;
     551            html += testObject.expected;
    570552        else if (tableId == 'crash-tests-table')
    571553            html += resultLink(stripExtension(test), '-stack.txt', 'crash stack');
     
    574556            html += textResultLinks(stripExtension(test));
    575557        } else if (tableId == 'new-tests-table') {
    576             var testObject = testObjectFor(test);
    577558            if (testObject.is_missing_audio)
    578559                html += resultLink(stripExtension(test), '-actual.wav', 'audio result');
     
    911892document.addEventListener('mousedown', handleMouseDown, false);
    912893
     894function failingTestsTable(tests, title, id)
     895{
     896    if (!tests.length)
     897        return '';
     898
     899    var html = '<p>' + title + ':</p>' +
     900        '<table id="' + id + '"><thead><tr>' +
     901        '<th>test</th>' +
     902        '<th id="text-results-header">results</th>' +
     903        '<th id="image-results-header">image results</th>';
     904
     905    if (globalState().results.uses_expectations_file)
     906        html += '<th>actual failure</th><th>expected failure</th>';
     907
     908    if (id == 'flaky-tests-table')
     909        html += '<th>failures</th>';
     910
     911    html += '</tr></thead>';
     912
     913    for (var i = 0; i < tests.length; i++)
     914        html += tableRow(tests[i]);
     915
     916    return html + '</table>';
     917}
     918
     919
    913920function generatePage()
    914921{
     922    forEachTest(processGlobalStateFor);
     923
    915924    var html = '<div id=toolbar>' +
    916925        '<a href="javascript:void()" onclick="expandAllExpectations()">expand all</a> ' +
     
    922931        '</div></div>';
    923932
    924     var tableRowsHtml = tableRows();
    925 
    926     if (tableRowsHtml) {
    927         html += '<p>Tests where results did not match expected results:</p>' +
    928             '<table id="results-table"><thead><tr>' +
    929             '<th>test</th>' +
    930             '<th id="text-results-header">results</th>' +
    931             '<th id="image-results-header">image results</th>';
    932 
    933         if (globalState().results.uses_expectations_file)
    934             html += '<th>actual failure</th><th>expected failure</th>';
    935 
    936         html += '</tr></thead>' + tableRowsHtml + '</table>';
    937     }
     933    html += failingTestsTable(globalState().nonFlakyFailingTests,
     934        'Tests where results did not match expected results', 'results-table');
     935
     936    html += failingTestsTable(globalState().flakyTests,
     937        'Flaky tests (failed the first run and got a different result on retry)', 'flaky-tests-table');
    938938
    939939    if (globalState().crashTests.length)
Note: See TracChangeset for help on using the changeset viewer.