Changeset 90442 in webkit

Timestamp:

Jul 6, 2011 12:39:30 AM (13 years ago)

Author:

abarth@webkit.org

Message:

2011-07-06 Adam Barth <​abarth@webkit.org>

garden-o-matic shouldn't show flaky tests by default
​https://bugs.webkit.org/show_bug.cgi?id=63979

Reviewed by Ojan Vafai.

If any of the actual test results are PASS, the test is flaky and we
can ignore it (at least for now). If we get awesome at dealing with
failing tests, we might want to tackle flaky tests by refering the user
to the flakiness dashboard, which is way more awesome at dealing with
flaky tests that this tool ever will be.

• Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (modified) (3 diffs)
• Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-07-06  Adam Barth  <abarth@webkit.org>
+
+        garden-o-matic shouldn't show flaky tests by default
+        https://bugs.webkit.org/show_bug.cgi?id=63979
+
+        Reviewed by Ojan Vafai.
+
+        If any of the actual test results are PASS, the test is flaky and we
+        can ignore it (at least for now).  If we get awesome at dealing with
+        failing tests, we might want to tackle flaky tests by refering the user
+        to the flakiness dashboard, which is way more awesome at dealing with
+        flaky tests that this tool ever will be.
+
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
+
 2011-07-05  Adam Barth  <abarth@webkit.org>
 

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js

@@ -7 +7 @@
 var kResultsName = 'full_results.json';
 var kMasterName = 'ChromiumWebkit';
-var kFailingResults = ['TIMEOUT', 'TEXT', 'CRASH', 'IMAGE','IMAGE+TEXT'];
+
+var PASS = 'PASS';
+var TIMEOUT = 'TIMEOUT';
+var TEXT = 'TEXT';
+var CRASH = 'CRASH';
+var IMAGE = 'IMAGE';
+var IMAGE_TEXT = 'IMAGE+TEXT';
+
+var kFailingResults = [TIMEOUT, TEXT, CRASH, IMAGE, IMAGE_TEXT];
 
 function isFailure(result)
@@ -14 +22 @@
 }
 
+function isSuccess(result)
+{
+    return result === PASS;
+}
+
 function anyIsFailure(resultsList)
 {
     return $.grep(resultsList, isFailure).length > 0;
+}
+
+function anyIsSuccess(resultsList)
+{
+    return $.grep(resultsList, isSuccess).length > 0;
 }
 
@@ -38 +56 @@
 function isUnexpectedFailure(resultNode)
 {
+    if (anyIsSuccess(resultNode.actual.split(' ')))
+        return false;
     return anyIsFailure(unexpectedResults(resultNode));
 }

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js

@@ -7 +7 @@
                 "expected": "IMAGE",
                 "actual": "IMAGE"
+            },
+            "flaky-scrollbarhtml": {
+                "expected": "PASS",
+                "actual": "PASS TEXT"
             }
         },