Changeset 90879 in webkit

Timestamp:

Jul 12, 2011 7:34:54 PM (13 years ago)

Author:

abarth@webkit.org

Message:

garden-o-matic should display how many times we've seen a failure
​https://bugs.webkit.org/show_bug.cgi?id=64417

Reviewed by Ojan Vafai.

This patch adds some UI to display how many times we've seen a given
failure, which can be helpful for determining whether that failure is a
real failure or a flaky test.

When a failure has only been seen once (i.e., only a single run on a
single bot), we set the opacity of to 50% to avoid distracting the
gardener.

This patch also refactors the failure walker to have a simpler API
internally by moving from an object-oriented paradigm to a functional
paradigm.

• Scripts/webkitpy/tool/servers/data/gardeningserver/main.css:
• Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/main.css (modified) (2 diffs)
• Scripts/webkitpy/tool/servers/data/gardeningserver/main.js (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (modified) (4 diffs)
• Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (modified) (2 diffs)
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js (modified) (2 diffs)
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-07-12  Adam Barth  <abarth@webkit.org>
+
+        garden-o-matic should display how many times we've seen a failure
+        https://bugs.webkit.org/show_bug.cgi?id=64417
+
+        Reviewed by Ojan Vafai.
+
+        This patch adds some UI to display how many times we've seen a given
+        failure, which can be helpful for determining whether that failure is a
+        real failure or a flaky test.
+
+        When a failure has only been seen once (i.e., only a single run on a
+        single bot), we set the opacity of to 50% to avoid distracting the
+        gardener.
+
+        This patch also refactors the failure walker to have a simpler API
+        internally by moving from an object-oriented paradigm to a functional
+        paradigm.
+
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/main.css:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:
+
 2011-07-12  Chris Rogers  <crogers@google.com>
 

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.css

@@ -39 +39 @@
 .butterbar .hide {
     margin-left: 20px;
+}
+
+/* If we've only seen a given test failure once, we dim it so as not to distract the gardener. */
+.test[data-failure-count="1"] {
+    opacity: 0.5;
 }
 
@@ -125 +130 @@
     color: #555;
 }
+
+.regression .when, .regression .how-many {
+    padding: 0px 2px;
+    display: inline-block;
+}
+
+.regression .how-many {
+    color: #555;
+}

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js

@@ -30 +30 @@
                 var builderNameList = base.keys(resultNodesByBuilder);
                 results.unifyRegressionRanges(builderNameList, testName, function(oldestFailingRevision, newestPassingRevision) {
-                    $('.when', regressions).append(ui.summarizeRegressionRange(oldestFailingRevision, newestPassingRevision));
+                    $('.when', testSummary).append(ui.summarizeRegressionRange(oldestFailingRevision, newestPassingRevision));
+                });
+                results.countFailureOccurances(builderNameList, testName, function(failureCount) {
+                    $(testSummary).attr('data-failure-count', failureCount);
+                    $('.how-many', testSummary).text(ui.failureCount(failureCount));
                 });
             });

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js

@@ -139 +139 @@
 function isUnexpectedFailure(resultNode)
 {
+    if (!resultNode)
+        return false;
     if (anyIsSuccess(resultNode.actual.split(' ')))
         return false;
@@ -183 +185 @@
 };
 
-function TestHistoryWalker(builderName, testName)
-{
-    this._builderName = builderName;
-    this._testName = testName;
-    this._indexOfNextKeyToFetch = 0;
-    this._keyList = [];
-}
-
-TestHistoryWalker.prototype.init = function(callback)
-{
-    var self = this;
-
-    base.jsonp(directoryOfResultsSummaryURL(self._builderName, kResultsName), function(keyList) {
-        self._keyList = keyList.map(function (element) { return element.key; });
-        callback();
-    });
-};
-
-TestHistoryWalker.prototype._fetchNextResultNode = function(callback)
-{
-    var self = this;
-
-    if (self._indexOfNextKeyToFetch >= self._keyList) {
-        callback(0, null);
-        return;
+function walkHistory(builderName, testName, callback)
+{
+    var indexOfNextKeyToFetch = 0;
+    var keyList = [];
+
+    function continueWalk()
+    {
+        if (indexOfNextKeyToFetch >= keyList.length) {
+            processResultNode(0, null);
+            return;
+        }
+
+        var key = keyList[indexOfNextKeyToFetch];
+        ++indexOfNextKeyToFetch;
+        g_resultsCache.get(key, function(resultsTree) {
+            var resultNode = results.resultNodeForTest(resultsTree, testName);
+            var revision = parseInt(resultsTree['revision'])
+            if (isNaN(revision))
+                revision = 0;
+            processResultNode(revision, resultNode);
+        });
     }
 
-    var key = self._keyList[self._indexOfNextKeyToFetch];
-    ++self._indexOfNextKeyToFetch;
-    g_resultsCache.get(key, function(resultsTree) {
-        var resultNode = results.resultNodeForTest(resultsTree, self._testName);
-        var revision = parseInt(resultsTree['revision'])
-        if (isNaN(revision))
-            revision = 0;
-        callback(revision, resultNode);
-    });
-};
-
-TestHistoryWalker.prototype.walkHistory = function(callback)
-{
-    var self = this;
-    self._fetchNextResultNode(function(revision, resultNode) {
+    function processResultNode(revision, resultNode)
+    {
         var shouldContinue = callback(revision, resultNode);
         if (!shouldContinue)
             return;
-        self.walkHistory(callback);
+        continueWalk();
+    }
+
+    base.jsonp(directoryOfResultsSummaryURL(builderName, kResultsName), function(directory) {
+        keyList = directory.map(function (element) { return element.key; });
+        continueWalk();
     });
 }
@@ -237 +227 @@
     var newestPassingRevision = 0;
 
-    var historyWalker = new TestHistoryWalker(builderName, testName);
-    historyWalker.init(function() {
-        historyWalker.walkHistory(function(revision, resultNode) {
-            if (!resultNode) {
-                newestPassingRevision = revision;
-                callback(oldestFailingRevision, newestPassingRevision);
-                return false;
-            }
-            if (isUnexpectedFailure(resultNode)) {
-                oldestFailingRevision = revision;
-                return true;
-            }
-            if (!oldestFailingRevision)
-                return true;  // We need to keep looking for a failing revision.
+    walkHistory(builderName, testName, function(revision, resultNode) {
+        if (!resultNode) {
             newestPassingRevision = revision;
             callback(oldestFailingRevision, newestPassingRevision);
             return false;
-        });
+        }
+        if (isUnexpectedFailure(resultNode)) {
+            oldestFailingRevision = revision;
+            return true;
+        }
+        if (!oldestFailingRevision)
+            return true;  // We need to keep looking for a failing revision.
+        newestPassingRevision = revision;
+        callback(oldestFailingRevision, newestPassingRevision);
+        return false;
     });
 };
@@ -298 +285 @@
                 callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
             }
+        });
+    });
+};
+
+results.countFailureOccurances = function(builderNameList, testName, callback)
+{
+    var queriesInFlight = builderNameList.length;
+    if (!queriesInFlight)
+        callback(0);
+
+    var failureCount = 0;
+    $.each(builderNameList, function(index, builderName) {
+        walkHistory(builderName, testName, function(revision, resultNode) {
+            if (isUnexpectedFailure(resultNode)) {
+                ++failureCount;
+                return true;
+            }
+
+            --queriesInFlight;
+            if (!queriesInFlight)
+                callback(failureCount);
+            return false;
         });
     });

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js

@@ -138 +138 @@
 }
 
-test("regressionRangeForFailure", 5, function() {
+test("walkHistory", 6, function() {
     var simulator = new NetworkSimulator();
 
@@ -246 +246 @@
             equals(newestPassingRevision, 90425);
         });
+
+        results.countFailureOccurances(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(failureCount) {
+            equals(failureCount, 4);
+        });
     });
 });

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js

@@ -34 +34 @@
           '<ul class="where"></ul>' +
           '<div class="when"></div>' +
+          '<div class="how-many"></div>' +
         '</div>');
     $('.what a', block).text(testName).attr('href', ui.urlForTest(testName)).attr('class', unexpectedResults.join(' '));
@@ -62 +63 @@
 };
 
+ui.failureCount = function(failureCount)
+{
+    if (failureCount < 1)
+        return '';
+    if (failureCount == 1)
+        return '(Seen once.)';
+    return '(Seen ' + failureCount + ' times.)';
+};
+
 ui.results = function(resultsURLs)
 {

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js

@@ -39 +39 @@
 });
 
+test("failureCount", 4, function() {
+    equal(ui.failureCount(0), '');
+    equal(ui.failureCount(1), '(Seen once.)');
+    equal(ui.failureCount(2), '(Seen 2 times.)');
+    equal(ui.failureCount(3), '(Seen 3 times.)');
+});
+
 test("results", 1, function() {
     var testResults = ui.results([