Changeset 90489 in webkit


Ignore:
Timestamp:
Jul 6, 2011 1:42:14 PM (13 years ago)
Author:
Adam Roben
Message:

Teach TestFailures how to load, parse, and interpret NRWT test results

Fixes <​http://webkit.org/b/61877> TestFailures page doesn't show testers that use
new-run-webkit-tests

Reviewed by Adam Barth.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:

(Builder.prototype.failureDiagnosisTextAndURL): Added support for the new 'flaky' failure
type. For now we don't account for image-only flakes (but TestFailures doesn't deal with
pixel tests at all currently).
(Builder.prototype.getNumberOfFailingTests): Relaxed the regex used for parsing the number
of failing tests from the buildbot output. Make sure not to count "new passes" (a new
category introduced by NRWT) as failures.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:

(LayoutTestResultsLoader.prototype.start): Moved a bunch of code from here to
_fetchAndParseORWTResults. This function now attempts to load NRWT results, then falls back
to loading ORWT results.
(LayoutTestResultsLoader.prototype._fetchAndParseNRWTResults): Added. Tries to load and
parse the full_results.json file for the given build.
(LayoutTestResultsLoader.prototype._fetchAndParseORWTResults): Added. Code came from start.
Fixed a bug along the way where we were sometimes calling the error callback instead of the
success callback when all tests passed.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js: Added.

(NRWTResultsParser): Do-nothing constructor.
(NRWTResultsParser.prototype.parse): Uses eval() (eww!) to get the test results out of the
JS string, then iterates over all the tests in the results data and builds up a data
structure matching what ORWTResultsParser returns.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js:

(Array.prototype.contains):
(String.prototype.contains):
Added these simple helper functions.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html: Pull in

NRWTResultsParser.js.

Location:
trunk/Tools
Files:
1 added
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js

    r90165 r90489  
    4242                url: urlStem + '-pretty-diff.html',
    4343            },
     44            flaky: {
     45                text: 'pretty diff (flaky)',
     46                url: urlStem + '-pretty-diff.html',
     47            },
    4448            timeout: {
    4549                text: 'timed out',
    … …  
    137141
    138142            result.failureCount = layoutTestStep.results[1].reduce(function(sum, outputLine) {
    139                 var match = /^(\d+) test case/.exec(outputLine);
     143                var match = /^(\d+)/.exec(outputLine);
    140144                if (!match)
    141145                    return sum;
    142                 // Don't count new tests as failures.
    143                 if (outputLine.indexOf('were new') >= 0)
     146                // Don't count new tests or passes as failures.
     147                if (outputLine.contains('were new') || outputLine.contains('new passes'))
    144148                    return sum;
    145149                return sum + parseInt(match[1], 10);

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js

    r90488 r90489  
    4242        var result = { tests: {}, tooManyFailures: false, version: currentCachedDataVersion };
    4343
     44        function cacheParseResultsAndCallCallback(parseResult) {
     45            result.tests = parseResult.tests;
     46            result.tooManyFailures = parseResult.tooManyFailures;
     47
     48            PersistentCache.set(cacheKey, result);
     49            callback(result.tests, result.tooManyFailures);
     50        }
     51
     52        var self = this;
     53        self._fetchAndParseNRWTResults(buildName, cacheParseResultsAndCallCallback, function() {
     54            self._fetchAndParseORWTResults(buildName, cacheParseResultsAndCallCallback, function() {
     55                // We couldn't fetch results for this build.
     56                PersistentCache.set(cacheKey, result);
     57                errorCallback(result.tests, result.tooManyFailures);
     58            });
     59        });
     60    },
     61
     62    _fetchAndParseNRWTResults: function(buildName, successCallback, errorCallback) {
     63        getResource(this._builder.resultsDirectoryURL(buildName) + 'full_results.json', function(xhr) {
     64            successCallback((new NRWTResultsParser()).parse(xhr.responseText));
     65        },
     66        function(xhr) {
     67            errorCallback();
     68        });
     69    },
     70
     71    _fetchAndParseORWTResults: function(buildName, successCallback, errorCallback) {
    4472        var parsedBuildName = this._builder.buildbot.parseBuildName(buildName);
    4573
    … …  
    4775        var resultsHTMLSupportsTooManyFailuresInfo = parsedBuildName.revision >= 89610;
    4876
     77        var result = { tests: {}, tooManyFailures: false };
     78
    4979        var self = this;
    5080
    51         function fetchAndParseResultsHTMLAndCallCallback(callback) {
     81        function fetchAndParseResultsHTMLAndCallCallback() {
    5282            getResource(self._builder.resultsPageURL(buildName), function(xhr) {
    5383                var parseResult = (new ORWTResultsParser()).parse(xhr.responseText);
    … …  
    5686                    result.tooManyFailures = parseResult.tooManyFailures;
    5787
    58                 PersistentCache.set(cacheKey, result);
    59                 callback(result.tests, result.tooManyFailures);
     88                successCallback(result);
    6089            },
    6190            function(xhr) {
    6291                // We failed to fetch results.html. run-webkit-tests must have aborted early.
    63                 PersistentCache.set(cacheKey, result);
    64                 errorCallback(result.tests, result.tooManyFailures);
     92                errorCallback();
    6593            });
    6694        }
    6795
    6896        if (resultsHTMLSupportsTooManyFailuresInfo) {
    69             fetchAndParseResultsHTMLAndCallCallback(callback);
     97            fetchAndParseResultsHTMLAndCallCallback();
    7098            return;
    7199        }
    … …  
    76104            if (failingTestCount < 0) {
    77105                // The number of failing tests couldn't be determined.
    78                 PersistentCache.set(cacheKey, result);
    79                 errorCallback(result.tests, result.tooManyFailures);
     106                errorCallback();
    80107                return;
    81108            }
    … …  
    83110            if (!failingTestCount) {
    84111                // All tests passed.
    85                 PersistentCache.set(cacheKey, result);
    86                 errorCallback(result.tests, result.tooManyFailures);
     112                successCallback(result);
    87113                return;
    88114            }
    89115
    90116            // Find out which tests failed.
    91             fetchAndParseResultsHTMLAndCallCallback(callback);
     117            fetchAndParseResultsHTMLAndCallCallback();
    92118        });
    93119    },

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js

    r90264 r90489  
    115115}
    116116
     117Array.prototype.contains = function(value) {
     118    return this.indexOf(value) >= 0;
     119}
     120
    117121Array.prototype.findFirst = function(predicate) {
    118122    for (var i = 0; i < this.length; ++i) {
    … …  
    146150        this.removeChild(this.firstChild);
    147151}
     152
     153String.prototype.contains = function(substring) {
     154    return this.indexOf(substring) >= 0;
     155}

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html

    r90488 r90489  
    3434    <script src="LayoutTestHistoryAnalyzer.js"></script>
    3535    <script src="LayoutTestResultsLoader.js"></script>
     36    <script src="NRWTResultsParser.js"></script>
    3637    <script src="ORWTResultsParser.js"></script>
    3738    <script src="PersistentCache.js"></script>

  • trunk/Tools/ChangeLog

    r90488 r90489  
     12011-07-06  Adam Roben  <aroben@apple.com>
     2
     3        Teach TestFailures how to load, parse, and interpret NRWT test results
     4
     5        Fixes <http://webkit.org/b/61877> TestFailures page doesn't show testers that use
     6        new-run-webkit-tests
     7
     8        Reviewed by Adam Barth.
     9
     10        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
     11        (Builder.prototype.failureDiagnosisTextAndURL): Added support for the new 'flaky' failure
     12        type. For now we don't account for image-only flakes (but TestFailures doesn't deal with
     13        pixel tests at all currently).
     14        (Builder.prototype.getNumberOfFailingTests): Relaxed the regex used for parsing the number
     15        of failing tests from the buildbot output. Make sure not to count "new passes" (a new
     16        category introduced by NRWT) as failures.
     17
     18        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
     19        (LayoutTestResultsLoader.prototype.start): Moved a bunch of code from here to
     20        _fetchAndParseORWTResults. This function now attempts to load NRWT results, then falls back
     21        to loading ORWT results.
     22        (LayoutTestResultsLoader.prototype._fetchAndParseNRWTResults): Added. Tries to load and
     23        parse the full_results.json file for the given build.
     24        (LayoutTestResultsLoader.prototype._fetchAndParseORWTResults): Added. Code came from start.
     25        Fixed a bug along the way where we were sometimes calling the error callback instead of the
     26        success callback when all tests passed.
     27
     28        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js: Added.
     29        (NRWTResultsParser): Do-nothing constructor.
     30        (NRWTResultsParser.prototype.parse): Uses eval() (eww!) to get the test results out of the
     31        JS string, then iterates over all the tests in the results data and builds up a data
     32        structure matching what ORWTResultsParser returns.
     33
     34        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js:
     35        (Array.prototype.contains):
     36        (String.prototype.contains):
     37        Added these simple helper functions.
     38
     39        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html: Pull in
     40        NRWTResultsParser.js.
     41
    1422011-07-06  Adam Roben  <aroben@apple.com>
    243
Note: See TracChangeset for help on using the changeset viewer.