Changeset 90771 in webkit


Ignore:
Timestamp:
Jul 11, 2011 11:54:13 AM (13 years ago)
Author:
abarth@webkit.org
Message:

garden-o-matic should be able to determine which revisions caused a given failure
​https://bugs.webkit.org/show_bug.cgi?id=64189

Reviewed by Adam Roben.

Walking the failure history looking for failures turns out to be
slightly tricky because the network requests are asynchronous.
Currently we do all the fetches serially and our cache is unbounded.
We'll probably optimize both those parameters eventually.

This patch also generalizes some functionality in the unit testing
framework to make testing this sort of code easier.

  • Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
  • Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
  • Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
Location:
trunk/Tools
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r90770 r90771  
     12011-07-11  Adam Barth  <abarth@webkit.org>
     2
     3        garden-o-matic should be able to determine which revisions caused a given failure
     4        https://bugs.webkit.org/show_bug.cgi?id=64189
     5
     6        Reviewed by Adam Roben.
     7
     8        Walking the failure history looking for failures turns out to be
     9        slightly tricky because the network requests are asynchronous.
     10        Currently we do all the fetches serially and our cache is unbounded.
     11        We'll probably optimize both those parameters eventually.
     12
     13        This patch also generalizes some functionality in the unit testing
     14        framework to make testing this sort of code easier.
     15
     16        * Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
     17        * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
     18        * Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js:
     19
    1202011-07-11  Adam Barth  <abarth@webkit.org>
    221

  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js

    r90652 r90771  
    6565};
    6666
     67// jQuery makes jsonp requests somewhat ugly (which is fair given that they're
     68// terrible for security). We use this wrapper to make our lives slightly easier.
     69base.jsonp = function(url, onsuccess)
     70{
     71    $.ajax({
     72        url: url,
     73        dataType: 'jsonp',
     74        success: onsuccess
     75    });
     76};
     77
    6778})();

  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js

    r90652 r90771  
    44
    55var kTestResultsServer = 'http://test-results.appspot.com/';
     6var kTestResultsQuery = kTestResultsServer + 'testfile?'
    67var kTestType = 'layout-tests';
    78var kResultsName = 'full_results.json';
    … …  
    5859}
    5960
     61function resultsParameters(builderName, testName)
     62{
     63    return {
     64        builder: builderName,
     65        master: kMasterName,
     66        testtype: kTestType,
     67        name: name,
     68    };
     69}
     70
     71function resultsSummaryURL(builderName, testName)
     72{
     73    return kTestResultsQuery + $.param(resultsParameters(builderName, testName));
     74}
     75
     76function directoryOfResultsSummaryURL(builderName, testName)
     77{
     78    var parameters = resultsParameters(builderName, testName);
     79    parameters['dir'] = 1;
     80    return kTestResultsQuery + $.param(parameters);
     81}
     82
     83function ResultsCache()
     84{
     85    this._cache = {};
     86}
     87
     88ResultsCache.prototype._fetch = function(key, callback)
     89{
     90    var self = this;
     91
     92    var url = kTestResultsServer + 'testfile?key=' + key;
     93    base.jsonp(url, function (resultsTree) {
     94        self._cache[key] = resultsTree;
     95        callback(resultsTree);
     96    });
     97};
     98
     99// Warning! This function can call callback either synchronously or asynchronously.
     100// FIXME: Consider using setTimeout to make this method always asynchronous.
     101ResultsCache.prototype.get = function(key, callback)
     102{
     103    if (key in this._cache) {
     104        callback(this._cache[key]);
     105        return;
     106    }
     107    this._fetch(key, callback);
     108};
     109
     110var g_resultsCache = new ResultsCache();
     111
    60112function anyIsFailure(resultsList)
    61113{
    … …  
    99151results.BuilderResults = function(resultsJSON)
    100152{
    101     this.m_resultsJSON = resultsJSON;
     153    this._resultsJSON = resultsJSON;
    102154};
    103155
    104156results.BuilderResults.prototype.unexpectedFailures = function()
    105157{
    106     return base.filterTree(this.m_resultsJSON.tests, isResultNode, isUnexpectedFailure);
     158    return base.filterTree(this._resultsJSON.tests, isResultNode, isUnexpectedFailure);
    107159};
    108160
    … …  
    111163    var unexpectedFailures = {};
    112164
    113     $.each(resultsByBuilder, function(buildName, builderResults) {
     165    $.each(resultsByBuilder, function(builderName, builderResults) {
    114166        $.each(builderResults.unexpectedFailures(), function(testName, resultNode) {
    115167            unexpectedFailures[testName] = unexpectedFailures[testName] || {};
    116             unexpectedFailures[testName][buildName] = resultNode;
     168            unexpectedFailures[testName][builderName] = resultNode;
    117169        });
    118170    });
    119171
    120172    return unexpectedFailures;
     173};
     174
     175function TestHistoryWalker(builderName, testName)
     176{
     177    this._builderName = builderName;
     178    this._testName = testName;
     179    this._indexOfNextKeyToFetch = 0;
     180    this._keyList = [];
     181}
     182
     183TestHistoryWalker.prototype.init = function(callback)
     184{
     185    var self = this;
     186
     187    base.jsonp(directoryOfResultsSummaryURL(self._builderName, kResultsName), function(keyList) {
     188        self._keyList = keyList.map(function (element) { return element.key; });
     189        callback();
     190    });
     191};
     192
     193TestHistoryWalker.prototype._fetchNextResultNode = function(callback)
     194{
     195    var self = this;
     196
     197    if (self._indexOfNextKeyToFetch >= self._keyList) {
     198        callback(0, null);
     199        return;
     200    }
     201
     202    var key = self._keyList[self._indexOfNextKeyToFetch];
     203    ++self._indexOfNextKeyToFetch;
     204    g_resultsCache.get(key, function(resultsTree) {
     205        var resultNode = results.resultNodeForTest(resultsTree, self._testName);
     206        callback(resultsTree['revision'], resultNode);
     207    });
     208};
     209
     210TestHistoryWalker.prototype.walkHistory = function(callback)
     211{
     212    var self = this;
     213    self._fetchNextResultNode(function(revision, resultNode) {
     214        var shouldContinue = callback(revision, resultNode);
     215        if (!shouldContinue)
     216            return;
     217        self.walkHistory(callback);
     218    });
     219}
     220
     221results.regressionRangeForFailure = function(builderName, testName, callback)
     222{
     223    var oldestFailingRevision = 0;
     224    var newestPassingRevision = 0;
     225
     226    var historyWalker = new TestHistoryWalker(builderName, testName);
     227    historyWalker.init(function() {
     228        historyWalker.walkHistory(function(revision, resultNode) {
     229            if (!resultNode) {
     230                newestPassingRevision = revision;
     231                callback(oldestFailingRevision, newestPassingRevision);
     232                return false;
     233            }
     234            if (isUnexpectedFailure(resultNode)) {
     235                oldestFailingRevision = revision;
     236                return true;
     237            }
     238            if (!oldestFailingRevision)
     239                return true;  // We need to keep looking for a failing revision.
     240            newestPassingRevision = revision;
     241            callback(oldestFailingRevision, newestPassingRevision);
     242            return false;
     243        });
     244    });
     245};
     246
     247results.resultNodeForTest = function(resultsTree, testName)
     248{
     249    var testNamePath = testName.split('/');
     250    var currentNode = resultsTree['tests'];
     251    $.each(testNamePath, function(index, segmentName) {
     252        if (!currentNode)
     253            return;
     254        currentNode = (segmentName in currentNode) ? currentNode[segmentName] : null;
     255    });
     256    return currentNode;
    121257};
    122258
    … …  
    190326};
    191327
    192 function resultsSummaryURL(builderName, name)
    193 {
    194     return kTestResultsServer + 'testfile' +
    195           '?builder=' + builderName +
    196           '&master=' + kMasterName +
    197           '&testtype=' + kTestType +
    198           '&name=' + name;
    199 }
    200 
    201328results.fetchResultsForBuilder = function(builderName, onsuccess)
    202329{
    203     $.ajax({
    204         url: resultsSummaryURL(builderName, kResultsName),
    205         dataType: 'jsonp',
    206         success: function(data) {
    207             onsuccess(new results.BuilderResults(data));
    208         }
     330    base.jsonp(resultsSummaryURL(builderName, kResultsName), function(resultsTree) {
     331        onsuccess(new results.BuilderResults(resultsTree));
    209332    });
    210333};
    … …  
    215338    var requestsInFlight = builderNameList.length;
    216339    $.each(builderNameList, function(index, builderName) {
    217         results.fetchResultsForBuilder(builderName, function(builderResults) {
    218             resultsByBuilder[builderName] = builderResults;
     340        results.fetchResultsForBuilder(builderName, function(resultsTree) {
     341            resultsByBuilder[builderName] = resultsTree;
    219342            --requestsInFlight;
    220343            if (!requestsInFlight)

  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js

    r90652 r90771  
    9494});
    9595
    96 test("fetchResultsURLs", 3, function() {
     96test("resultNodeForTest", 4, function() {
     97    deepEqual(results.resultNodeForTest(kExampleResultsJSON, "userscripts/another-test.html"), {
     98        "expected": "PASS",
     99        "actual": "TEXT"
     100    });
     101    equals(results.resultNodeForTest(kExampleResultsJSON, "foo.html"), null);
     102    equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo.html"), null);
     103    equals(results.resultNodeForTest(kExampleResultsJSON, "userscripts/foo/bar.html"), null);
     104});
     105
     106function NetworkSimulator()
     107{
     108    this._pendingCallbacks = [];
     109};
     110
     111NetworkSimulator.prototype.scheduleCallback = function(callback)
     112{
     113    this._pendingCallbacks.push(callback);
     114}
     115
     116NetworkSimulator.prototype.runTest = function(testCase)
     117{
     118    var self = this;
    97119    var realBase = window.base;
    98120
    99     var pendingCallbacks = {};
    100121    window.base = {};
    101     base.probe = function(url, options) {
    102         pendingCallbacks[url] = options;
    103     };
    104122    base.endsWith = realBase.endsWith;
    105123    base.trimExtension = realBase.trimExtension;
    106 
    107     results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", function(resultURLs) {
    108         deepEqual(resultURLs, [
    109             "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
    110             "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
    111             "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
    112         ]);
    113     });
     124    if (self.probeHook)
     125        base.probe = self.probeHook;
     126    if (self.jsonpHook)
     127        base.jsonp = self.jsonpHook;
     128
     129    testCase();
     130
     131    while (this._pendingCallbacks.length) {
     132        var callback = this._pendingCallbacks.shift();
     133        callback();
     134    }
     135
     136    window.base = realBase;
     137    equal(window.base, realBase, "Failed to restore real base!");
     138}
     139
     140test("regressionRangeForFailure", 3, function() {
     141    simulator = new NetworkSimulator();
     142
     143    var keyMap = {
     144        "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw": {
     145            "tests": {
     146                "userscripts": {
     147                    "another-test.html": {
     148                        "expected": "PASS",
     149                        "actual": "TEXT"
     150                    }
     151                },
     152            },
     153            "revision": "90430"
     154        },
     155        "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw":{
     156            "tests": {
     157                "userscripts": {
     158                    "user-script-video-document.html": {
     159                        "expected": "FAIL",
     160                        "actual": "TEXT"
     161                    },
     162                    "another-test.html": {
     163                        "expected": "PASS",
     164                        "actual": "TEXT"
     165                    }
     166                },
     167            },
     168            "revision": "90429"
     169        },
     170        "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw":{
     171            "tests": {
     172                "userscripts": {
     173                    "another-test.html": {
     174                        "expected": "PASS",
     175                        "actual": "TEXT"
     176                    }
     177                },
     178            },
     179            "revision": "90426"
     180        },
     181        "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw":{
     182            "tests": {
     183                "userscripts": {
     184                    "user-script-video-document.html": {
     185                        "expected": "FAIL",
     186                        "actual": "TEXT"
     187                    },
     188                },
     189            },
     190            "revision": "90424"
     191        }
     192    };
     193
     194    simulator.jsonpHook = function(url, callback) {
     195        simulator.scheduleCallback(function() {
     196            if (/dir=1/.test(url)) {
     197                callback([
     198                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGLncUAw" },
     199                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNfTUAw" },
     200                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGJWCUQw" },
     201                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGKbLUAw" },
     202                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGOj5UAw" },
     203                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGP-AUQw" },
     204                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGPL3UAw" },
     205                    { "key": "agx0ZXN0LXJlc3VsdHNyEAsSCFRlc3RGaWxlGNHJQAw" },
     206                ]);
     207            } else {
     208                var key = url.match(/key=([^&]+)/)[1];
     209                callback(keyMap[key]);
     210            }
     211        });
     212    };
     213    simulator.runTest(function() {
     214        results.regressionRangeForFailure("Mock Builder", "userscripts/another-test.html", function(oldestFailingRevision, newestPassingRevision) {
     215            equals(oldestFailingRevision, "90426");
     216            equals(newestPassingRevision, "90424");
     217        });
     218    });
     219});
     220
     221test("fetchResultsURLs", 3, function() {
     222    var simulator = new NetworkSimulator();
    114223
    115224    var probedURLs = [];
    116     for (var url in pendingCallbacks) {
    117         probedURLs.push(url);
    118         if (realBase.endsWith(url, '.txt'))
    119             pendingCallbacks[url].success.call();
    120         else
    121             pendingCallbacks[url].error.call();
    122     }
     225    simulator.probeHook = function(url, options)
     226    {
     227        simulator.scheduleCallback(function() {
     228            probedURLs.push(url);
     229            if (base.endsWith(url, '.txt'))
     230                options.success.call();
     231            else
     232                options.error.call();
     233        });
     234    };
     235
     236    simulator.runTest(function() {
     237        results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", function(resultURLs) {
     238            deepEqual(resultURLs, [
     239                "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
     240                "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
     241                "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
     242            ]);
     243        });
     244    });
    123245
    124246    deepEqual(probedURLs, [
    … …  
    130252        "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
    131253    ]);
    132 
    133     window.base = realBase;
    134     equal(window.base, realBase, "Failed to restore real base!");
    135 });
     254});
Note: See TracChangeset for help on using the changeset viewer.