Changeset 85346 in webkit

Timestamp:

Apr 29, 2011 12:14:42 PM (13 years ago)

Author:

ojan@chromium.org

Message:

2011-04-29 Ojan Vafai <​ojan@chromium.org>

Reviewed by Eric Seidel.

handle flaky failures
​https://bugs.webkit.org/show_bug.cgi?id=59820

Handle that actual can be a space-separated list of results
when we do retries. Also, show expected flaky tests that pass
in the unexpected passes table.

• fast/harness/resources/results-test.js:
• fast/harness/results.html:

Location:

trunk/LayoutTests

Files:

• ChangeLog (modified) (1 diff)
• fast/harness/resources/results-test.js (modified) (3 diffs)
• fast/harness/results.html (modified) (4 diffs)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2011-04-29  Ojan Vafai  <ojan@chromium.org>
+
+        Reviewed by Eric Seidel.
+
+        handle flaky failures
+        https://bugs.webkit.org/show_bug.cgi?id=59820
+
+        Handle that actual can be a space-separated list of results
+        when we do retries. Also, show expected flaky tests that pass
+        in the unexpected passes table.
+
+        * fast/harness/resources/results-test.js:
+        * fast/harness/results.html:
+
 2011-04-29  Mark Pilgrim  <pilgrim@chromium.org>
 

trunk/LayoutTests/fast/harness/resources/results-test.js

@@ -221 +221 @@
     runDefaultSingleRowTest('bar-flaky-fail-unexpected.html', 'PASS TEXT', 'IMAGE', false, '', 'expected actual diff ');
     runDefaultSingleRowTest('bar-crash.html', 'TEXT', 'CRASH', false, 'stack ', '');
-    runDefaultSingleRowTest('bar-audio.html', 'TEXT', 'AUDIO', false, 'expected actual ', '');
+    runDefaultSingleRowTest('bar-audio.html', 'TEXT', 'AUDIO', false, 'expected audio actual audio ', '');
     runDefaultSingleRowTest('bar-timeout.html', 'TEXT', 'TIMEOUT', false, 'expected actual diff ', '');
     runDefaultSingleRowTest('bar-image.html', 'TEXT', 'IMAGE', false, '', 'expected actual diff ');
@@ -238 +238 @@
     results = mockResults();
     var subtree = results.tests['foo'] = {}
-    subtree['bar-flaky-pass.html'] = mockExpectation('PASS FAIL', 'PASS');
-    runTest(results, function() {
-        // FIXME: should a PASS FAIL test that passes go in the unexpected passes table?
-        assertTrue(document.body.textContent.indexOf('foo/bar-flaky-pass.html') == -1);
+    subtree['bar-flaky-pass.html'] = mockExpectation('PASS TEXT', 'PASS');
+    runTest(results, function() {
+        assertTrue(!document.getElementById('results-table'));
+        assertTrue(document.getElementById('passes-table'));
+        assertTrue(document.body.textContent.indexOf('foo/bar-flaky-pass.html') != -1);
+    });
+
+    results = mockResults();
+    var subtree = results.tests['foo'] = {}
+    subtree['bar-flaky-fail.html'] = mockExpectation('PASS TEXT', 'IMAGE PASS');
+    runTest(results, function() {
+        assertTrue(document.getElementById('results-table'));
+        assertTrue(document.body.textContent.indexOf('bar-flaky-fail.html') != -1);
     });
 
@@ -357 +366 @@
     shouldUseTracLinks = oldShouldUseTracLinks;
 
+    runDefaultSingleRowTest('bar-flaky-crash.html', 'TEXT', 'TIMEOUT CRASH AUDIO', false, 'expected actual diff stack expected audio actual audio ', '');
+
     document.body.innerHTML = '<pre>' + g_log.join('\n') + '</pre>';
 }

trunk/LayoutTests/fast/harness/results.html

@@ -320 +320 @@
         return;
     }
-
+    
     var expected = testObject.expected || 'PASS';
-    testObject.isExpected = actual == 'SKIP';
-    if (!testObject.isExpected && globalState().results.uses_expectations_file) {
+    testObject.isExpected = true;
+    if (actual != 'SKIP' && globalState().results.uses_expectations_file) {
         var expectedArray = expected.split(' ');
-        if (expectedArray.indexOf(actual) != -1)
-            testObject.isExpected = true;
-        else if (expectedArray.indexOf('FAIL') != -1)
-            testObject.isExpected = actual == 'IMAGE' || actual == 'TEXT' || actual == 'IMAGE+TEXT';
-    }
-
-    if (actual == 'PASS' && expected != 'PASS' && !testObject.isExpected)
+        var actualArray = actual.split(' ');
+        for (var i = 0; i < actualArray.length; i++) {
+            var actualValue = actualArray[i];
+            if (expectedArray.indexOf(actualValue) == -1 &&
+                (expectedArray.indexOf('FAIL') == -1 ||
+                 (actualValue != 'IMAGE' && actualValue != 'TEXT' && actualValue != 'IMAGE+TEXT')))
+                testObject.isExpected = false;
+        }
+    }
+
+    if (actual == 'PASS' && expected != 'PASS')
         globalState().unexpectedPassTests.push(test);            
 }
@@ -337 +341 @@
 function tableRow(testObject)
 {    
+    var row = '<tbody'
+    if (globalState().results.uses_expectations_file)
+        row += ' class="' + (testObject.isExpected ? 'expected' : '') + '"';
+    row += '><tr>';
+
+    row += '<td>' + testLink(testObject.name) + '</td>';
+
+    var test_prefix = stripExtension(testObject.name);
+    row += '<td>';
+    
     var actual = testObject.actual;
-    if (actual == 'PASS')
-        return '';
-
-    var row = '<td>' + testLink(testObject.name) + '</td>';
-    var test_prefix = stripExtension(testObject.name);
-
-    row += '<td>';
-
-    if (actual == 'CRASH')
-        row += resultLink(test_prefix, '-stack.txt', 'stack');
-    else if (actual == 'AUDIO') {
-        row += resultLink(test_prefix, '-expected.wav', 'expected');
-        row += resultLink(test_prefix, '-actual.wav', 'actual');
-    } else if (actual.indexOf('TEXT') != -1 || actual == 'TIMEOUT') {
+    if (actual.indexOf('TEXT') != -1 || actual.indexOf('TIMEOUT') != -1) {
         // FIXME: only include timeout actual/expected results here if we actually spit out results for timeout tests.
         globalState().hasTextFailures = true;
@@ -365 +366 @@
     }
 
+    if (actual.indexOf('CRASH') != -1)
+        row += resultLink(test_prefix, '-stack.txt', 'stack');
+    
+    if (actual.indexOf('AUDIO') != -1) {
+        row += resultLink(test_prefix, '-expected.wav', 'expected audio');
+        row += resultLink(test_prefix, '-actual.wav', 'actual audio');
+    }
+
     row += '</td><td>';
 
@@ -389 +398 @@
       row += '<td>' + testObject.expected + '</td>';
 
-    return '<tbody class="' + (testObject.isExpected ? 'expected' : '') + '"><tr>' + row + '</tr></tbody>';
+    row += '</tr></tbody>';
+    return row;
 }