Changeset 90866 in webkit

Timestamp:

Jul 12, 2011 4:31:05 PM (13 years ago)

Author:

abarth@webkit.org

Message:

garden-o-matic should display regression ranges
​https://bugs.webkit.org/show_bug.cgi?id=64407

Reviewed by Dimitri Glazkov.

This patch computes a regression range for a failure by intersecting
the regression ranges seen by the various bots. We make the underlying
assumption that a test is only failing due to one revision at any given
moment. If that's not true, this code probably explodes.

The regression ranges appear asynchronously, which might be a jarring
UI. We'll have to experiment to see.

• Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/base.js (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/main.js (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/results.js (modified) (1 diff)
• Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js (modified) (4 diffs)
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js (modified) (3 diffs)
• Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-07-12  Adam Barth  <abarth@webkit.org>
+
+        garden-o-matic should display regression ranges
+        https://bugs.webkit.org/show_bug.cgi?id=64407
+
+        Reviewed by Dimitri Glazkov.
+
+        This patch computes a regression range for a failure by intersecting
+        the regression ranges seen by the various bots.  We make the underlying
+        assumption that a test is only failing due to one revision at any given
+        moment.  If that's not true, this code probably explodes.
+
+        The regression ranges appear asynchronously, which might be a jarring
+        UI.  We'll have to experiment to see.
+
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
+        * Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:
+
 2011-07-12  Adam Barth  <abarth@webkit.org>
 

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js

@@ -37 +37 @@
     });
     return result;
+};
+
+base.keys = function(dictionary)
+{
+    var keys = [];
+    $.each(dictionary, function(key, value) {
+        keys.push(key);
+    });
+    return keys;
 };
 

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js

@@ -38 +38 @@
     deepEqual(base.uniquifyArray(["a", "b", "b"]), ["a", "b"]);
     deepEqual(base.uniquifyArray(["a", "b", "b", "a"]), ["a", "b"]);
+});
+
+test("keys", 4, function() {
+    deepEqual(base.keys({}), []);
+    deepEqual(base.keys({"a": 1}), ["a"]);
+    deepEqual(base.keys({"a": 1, "b": 0}), ["a", "b"]);
+    deepEqual(base.keys({"a": 1, "b": { "c" : 1}}), ["a", "b"]);
 });
 

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js

@@ -23 +23 @@
             partyTime.fadeIn(1200).delay(7000).fadeOut();
         } else {
-            var resultsSummary = ui.summarizeResultsByTest(unexpectedFailures);
-            $('.results').append($(resultsSummary).addClass('regression'));
+            var regressions = $('<div class="results-summary regression"></div>');
+            $.each(resultsByTest, function(testName, resultNodesByBuilder) {
+                var testSummary = ui.summarizeTest(testName, resultNodesByBuilder);
+                regressions.append(testSummary);
+
+                var builderNameList = base.keys(resultNodesByBuilder);
+                results.unifyRegressionRanges(builderNameList, testName, function(oldestFailingRevision, newestPassingRevision) {
+                    $('.when', regressions).append(ui.summarizeRegressionRange(oldestFailingRevision, newestPassingRevision));
+                });
+            });
+            $('.results').append(regressions);
         }
         setIconState(hasFailures);

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js

@@ -258 +258 @@
 };
 
+function mergeRegressionRanges(regressionRanges)
+{
+    var mergedRange = {};
+
+    mergedRange.oldestFailingRevision = 0;
+    mergedRange.newestPassingRevision = 0;
+
+    $.each(regressionRanges, function(builderName, range) {
+        if (!mergedRange.oldestFailingRevision)
+            mergedRange.oldestFailingRevision = range.oldestFailingRevision;
+        if (!mergedRange.newestPassingRevision)
+            mergedRange.newestPassingRevision = range.newestPassingRevision;
+
+        if (range.oldestFailingRevision < mergedRange.oldestFailingRevision)
+            mergedRange.oldestFailingRevision = range.oldestFailingRevision;
+        if (range.newestPassingRevision > mergedRange.newestPassingRevision)
+            mergedRange.newestPassingRevision = range.newestPassingRevision;
+    });
+    return mergedRange;
+}
+
+results.unifyRegressionRanges = function(builderNameList, testName, callback)
+{
+    var queriesInFlight = builderNameList.length;
+    if (!queriesInFlight)
+        callback(0, 0);
+
+    var regressionRanges = {};
+    $.each(builderNameList, function(index, builderName) {
+        results.regressionRangeForFailure(builderName, testName, function(oldestFailingRevision, newestPassingRevision) {
+            var range = {};
+            range.oldestFailingRevision = oldestFailingRevision;
+            range.newestPassingRevision = newestPassingRevision;
+            regressionRanges[builderName] = range;
+
+            --queriesInFlight;
+            if (!queriesInFlight) {
+                var mergedRange = mergeRegressionRanges(regressionRanges);
+                callback(mergedRange.oldestFailingRevision, mergedRange.newestPassingRevision);
+            }
+        });
+    });
+};
+
 results.resultNodeForTest = function(resultsTree, testName)
 {

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js

@@ -138 +138 @@
 }
 
-test("regressionRangeForFailure", 3, function() {
+test("regressionRangeForFailure", 5, function() {
     var simulator = new NetworkSimulator();
 
@@ -189 +189 @@
             },
             "revision": "90424"
+        },
+        "abc":{
+            "tests": {
+                "userscripts": {
+                    "another-test.html": {
+                        "expected": "PASS",
+                        "actual": "TEXT"
+                    }
+                },
+            },
+            "revision": "90426"
+        },
+        "xyz":{
+            "tests": {
+            },
+            "revision": "90425"
         }
     };
@@ -195 +211 @@
         simulator.scheduleCallback(function() {
             if (/dir=1/.test(url)) {
-                callback([
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
-                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
-                ]);
+                if (/builder=Mock/.test(url)) {
+                    callback([
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
+                        { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
+                    ]);
+                } else if (/builder=Another/.test(url)) {
+                    callback([
+                        { "key": "abc" },
+                        { "key": "xyz" },
+                    ]);
+                } else {
+                    ok(false, 'Unexpected URL: ' + url);
+                }
             } else {
                 var key = url.match(/key=([^&]+)/)[1];
@@ -215 +240 @@
             equals(oldestFailingRevision, 90426);
             equals(newestPassingRevision, 90424);
+        });
+
+        results.unifyRegressionRanges(["Mock Builder", "Another Builder"], "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
+            equals(oldestFailingRevision, 90426);
+            equals(newestPassingRevision, 90425);
         });
     });

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js

@@ -13 +13 @@
 }
 
+ui.urlForRevisionRange = function(firstRevision, lastRevision)
+{
+    if (firstRevision != lastRevision)
+        return 'http://trac.webkit.org/log/trunk/?rev=' + firstRevision + '&stop_rev=' + lastRevision + '&limit=100&verbose=on';
+    return 'http://trac.webkit.org/changeset/' + firstRevision;
+};
+
 ui.summarizeTest = function(testName, resultNodesByBuilder)
 {
@@ -21 +28 @@
           '<span>fails on</span>' +
           '<ul class="where"></ul>' +
+          '<div class="when"></div>' +
         '</div>');
     $('.what a', block).text(testName).attr('href', ui.urlForTest(testName)).attr('class', unexpectedResults.join(' '));
@@ -32 +40 @@
 };
 
-ui.summarizeResultsByTest = function(resultsByTest)
+ui.summarizeRegressionRange = function(oldestFailingRevision, newestPassingRevision)
 {
-    var block = $('<div class="results-summary"></div>');
-    $.each(resultsByTest, function(testName, resultNodesByBuilder) {
-        block.append(ui.summarizeTest(testName, resultNodesByBuilder));
-    });
-    return block;
+    if (!oldestFailingRevision || !newestPassingRevision)
+        return $();
+
+    var impliedFirstFailingRevision = newestPassingRevision + 1;
+
+    var href = ui.urlForRevisionRange(impliedFirstFailingRevision, oldestFailingRevision);
+    var textForRevisionRange = impliedFirstFailingRevision == oldestFailingRevision ? impliedFirstFailingRevision : impliedFirstFailingRevision + ':' + oldestFailingRevision;
+    var text = 'Regression ' + textForRevisionRange;
+
+    return $('<a class="regression-range"></a>').attr('href', href).text(text);
 };
 

trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js

@@ -20 +20 @@
 }
 
-test("summarizeResultsByTest", 3, function() {
-    var resultsSummary = ui.summarizeResultsByTest(kExampleResultsByTest);
-    var resultsSummaryHTML = resultsSummary.html();
-    ok(resultsSummaryHTML.indexOf('scrollbars/custom-scrollbar-with-incomplete-style.html') != -1);
-    ok(resultsSummaryHTML.indexOf('userscripts/another-test.html') != -1);
-    ok(resultsSummaryHTML.indexOf('Mock Builder') != -1);
+test("summarizeTest", 3, function() {
+    var testName = 'userscripts/another-test.html';
+    var summary = ui.summarizeTest(testName, kExampleResultsByTest[testName]);
+    var summaryHTML = summary.html();
+    ok(summaryHTML.indexOf('scrollbars/custom-scrollbar-with-incomplete-style.html') == -1);
+    ok(summaryHTML.indexOf('userscripts/another-test.html') != -1);
+    ok(summaryHTML.indexOf('Mock Builder') != -1);
+});
+
+test("summarizeRegressionRange", 2, function() {
+    var summaryWithMultipleRevisions = ui.summarizeRegressionRange(90424, 90426);
+    summaryWithMultipleRevisions.wrap('<wrapper></wrapper>');
+    equal(summaryWithMultipleRevisions.parent().html(), '<a class="regression-range" href="http://trac.webkit.org/log/trunk/?rev=90427&amp;stop_rev=90424&amp;limit=100&amp;verbose=on">Regression 90427:90424</a>');
+
+    var summaryWithOneRevision = ui.summarizeRegressionRange(90425, 90426);
+    summaryWithOneRevision.wrap('<wrapper></wrapper>');
+    equal(summaryWithOneRevision.parent().html(), '<a class="regression-range" href="http://trac.webkit.org/log/trunk/?rev=90427&amp;stop_rev=90425&amp;limit=100&amp;verbose=on">Regression 90427:90425</a>');
 });