Changeset 116599 in webkit


Ignore:
Timestamp:
May 9, 2012 8:08:02 PM (12 years ago)
Author:
ojan@chromium.org
Message:

Change "losing test coverage" to cover all non-layout test problems in garden-o-matic
​https://bugs.webkit.org/show_bug.cgi?id=86043

Reviewed by Adam Barth.

-Get rid of the gtest iframe.
-Cover all non-layout test problems. Before we wouldn't catch compile failures
or gtest failures since they were not on the webkit testing bots.
-Include the list of failing steps next to the builder name. This is a lot less
cluttered than you'd expect and already helped me find a compile failure early.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders_unittests.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/config.js:

Delete unused map.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/controllers.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/garden-o-matic.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures_unittests.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications_unittests.js:
  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui_unittests.js:
Location:
trunk/Tools
Files:
12 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders.js

    r97192 r116599  
    4242}
    4343
    44 function isStepRequredForTestCoverage(step)
    45 {
    46     switch(step.name) {
    47     case kUpdateStepName:
    48     case kUpdateScriptsStepName:
    49     case kCompileStepName:
    50     case kWebKitTestsStepName:
    51         return true;
    52     default:
    53         return false;
    54     }
    55 }
    56 
    5744function didFail(step)
    5845{
    … …  
    6148        return step.text.indexOf(kCrashedOrHungOutputMarker) != -1;
    6249    }
    63     // FIXME: Is this the correct way to test for failure?
    6450    return step.results[0] > 0;
    6551}
    6652
    67 function didFailStepRequredForTestCoverage(buildInfo)
     53function failingSteps(buildInfo)
    6854{
    69     return buildInfo.steps.filter(isStepRequredForTestCoverage).filter(didFail).length > 0;
     55    return buildInfo.steps.filter(didFail);
    7056}
    7157
    … …  
    9177function fetchMostRecentBuildInfoByBuilder(callback)
    9278{
    93     var buildInfoByBuilder = {};
    94     var builderNames = Object.keys(config.kBuilders);
    95     var requestTracker = new base.RequestTracker(builderNames.length, callback, [buildInfoByBuilder]);
    9679    net.get(kChromiumBuildBotURL + '/json/builders', function(builderStatus) {
    97         $.each(builderNames, function(index, builderName) {
     80        var buildInfoByBuilder = {};
     81        var builderNames = Object.keys(builderStatus);
     82        var requestTracker = new base.RequestTracker(builderNames.length, callback, [buildInfoByBuilder]);
     83        builderNames.forEach(function(builderName) {
     84            // FIXME: Should garden-o-matic show these? I can imagine showing the deps bots being useful at least so
     85            // that the gardener only need to look at garden-o-matic and never at the waterfall. Not really sure who
     86            // watches the GPU bots.
     87            if (builderName.indexOf('GPU') != -1 || builderName.indexOf('deps') != -1) {
     88                requestTracker.requestComplete();
     89                return;
     90            }
     91
    9892            var buildNumber = mostRecentCompletedBuildNumber(builderStatus[builderName]);
    9993            if (!buildNumber) {
    … …  
    111105}
    112106
    113 builders.buildersFailingStepRequredForTestCoverage = function(callback)
     107builders.buildersFailingNonLayoutTests = function(callback)
    114108{
    115109    fetchMostRecentBuildInfoByBuilder(function(buildInfoByBuilder) {
    116         var builderNameList = [];
     110        var failureList = {};
    117111        $.each(buildInfoByBuilder, function(builderName, buildInfo) {
    118112            if (!buildInfo)
    119113                return;
    120             if (didFailStepRequredForTestCoverage(buildInfo))
    121                 builderNameList.push(builderName);
     114            var failures = failingSteps(buildInfo);
     115            if (failures.length)
     116                failureList[builderName] = failures.map(function(failure) { return failure.name; });
    122117        });
    123         callback(builderNameList);
     118        callback(failureList);
    124119    });
    125120};

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders_unittests.js

    r104423 r116599  
    583583};
    584584
    585 test("buildersFailingStepRequredForTestCoverage", 3, function() {
     585test("buildersFailing", 3, function() {
    586586    var simulator = new NetworkSimulator();
    587587
    … …  
    609609
    610610    simulator.runTest(function() {
    611         builders.buildersFailingStepRequredForTestCoverage(function(builderNameList) {
    612             deepEqual(builderNameList, ["Webkit Mac10.6"]);
     611        builders.buildersFailingNonLayoutTests(function(builderNameList) {
     612            deepEqual(builderNameList, {
     613                "Webkit Linux": [
     614                    "webkit_gpu_tests"
     615                ],
     616                "Webkit Mac10.6": [
     617                    "compile",
     618                    "webkit_gpu_tests"
     619                ]
     620            });
    613621        });
    614622    });
    … …  
    621629});
    622630
    623 test("buildersFailingStepRequredForTestCoverage (run-webkit-tests crash)", 3, function() {
     631test("buildersFailing (run-webkit-tests crash)", 3, function() {
    624632    var simulator = new NetworkSimulator();
    625633
    … …  
    649657
    650658    simulator.runTest(function() {
    651         builders.buildersFailingStepRequredForTestCoverage(function(builderNameList) {
    652             deepEqual(builderNameList, ["Webkit Linux"]);
     659        builders.buildersFailingNonLayoutTests(function(builderNameList) {
     660            deepEqual(builderNameList, {
     661                "Webkit Linux": [
     662                    "extract_build",
     663                    "webkit_tests",
     664                    "archive_webkit_tests_results",
     665                    "webkit_gpu_tests",
     666                    "archive_webkit_tests_gpu_results"
     667                ]
     668            });
    653669        });
    654670    });

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/config.js

    r110739 r116599  
    4545};
    4646
    47 config.kBuildersThatOnlyCompile = [
    48     'Webkit Win Builder',
    49     'Webkit Win Builder (dbg)',
    50     // FIXME: Where is the Linux Builder?
    51     'Webkit Mac Builder',
    52     'Webkit Mac Builder (dbg)',
    53     'Win Builder',
    54 ];
    55 
    5647config.kTracURL = 'http://trac.webkit.org';
    5748config.kBugzillaURL = 'https://bugs.webkit.org';

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/controllers.js

    r112149 r116599  
    288288        this._notification = null;
    289289    },
    290     update: function(builderNameList)
    291     {
    292         if (builderNameList.length == 0) {
     290    update: function(failuresList)
     291    {
     292        if (Object.keys(failuresList).length == 0) {
    293293            if (this._notification) {
    294294                this._notification.dismiss();
    … …  
    303303        // FIXME: We should provide regression ranges for the failing builders.
    304304        // This doesn't seem to happen often enough to worry too much about that, however.
    305         this._notification.setFailingBuilders(builderNameList);
     305        this._notification.setFailingBuilders(failuresList);
    306306    }
    307307});

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/garden-o-matic.js

    r116483 r116599  
    3535var g_failuresController = null;
    3636
    37 var g_losingTestCoverageBuilders = null;
     37var g_nonLayoutTestFailureBuilders = null;
    3838
    3939function update()
    … …  
    5252    g_info.add(updating);
    5353
    54     builders.buildersFailingStepRequredForTestCoverage(g_losingTestCoverageBuilders.update.bind(g_losingTestCoverageBuilders));
     54    builders.buildersFailingNonLayoutTests(g_nonLayoutTestFailureBuilders.update.bind(g_nonLayoutTestFailureBuilders));
    5555
    5656    base.callInParallel([model.updateRecentCommits, model.updateResultsByBuilder], function() {
    … …  
    9191        {
    9292            var resultsContainer = onebar.results();
    93             console.log(resultsContainer);
    9493            $(resultsContainer).empty().append(resultsView);
    9594            onebar.select('results');
    … …  
    101100
    102101    g_info = new ui.notifications.Stream();
    103     g_losingTestCoverageBuilders = new controllers.FailingBuilders(g_info, 'Losing test coverage');
     102    g_nonLayoutTestFailureBuilders = new controllers.FailingBuilders(g_info, 'Non-layout test failures');
    104103
    105104    // FIXME: This should be an Action object.

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui.js

    r116483 r116599  
    5656}
    5757
    58 ui.urlForChromiumGtestSummary = function()
    59 {
    60     return 'http://build.chromium.org/p/chromium.webkit/horizontal_one_box_per_builder?' +
    61         'builder=Win%20Builder' +
    62         '&builder=Win%20Reliability%20Builder' +
    63         '&builder=Vista%20Tests&builder=Win%20Reliability' +
    64         '&builder=Win+(dbg)' +
    65         '&builder=Win%20Shared%20Builder%20%28dbg%29' +
    66         '&builder=Mac10.6%20Tests' +
    67         '&builder=Mac+Builder+(dbg)' +
    68         '&builder=Linux%20Tests' +
    69         '&builder=Linux%20Valgrind' +
    70         '&builder=Vista%20Perf' +
    71         '&builder=Linux%20Perf' +
    72         '&builder=Mac10.6%20Perf';
    73 }
    74 
    7558ui.rolloutReasonForTestNameList = function(testNameList)
    7659{
    … …  
    9275            '<div id="unexpected"></div>' +
    9376            '<div id="expected"></div>' +
    94             '<div id="results"></div>' +
    95             '<div id="chromium-gtests"><span id="gtest-label">Chromium gtest bots:</span>' +
    96                 '<iframe scrolling="no" src="' + ui.urlForChromiumGtestSummary() + '"></iframe>' +
    97             '</div>';
     77            '<div id="results"></div>';
    9878        this._tabNames = [
    9979            'unexpected',
    … …  
    10181            'results',
    10282        ]
     83
    10384        this._tabIndexToSavedScrollOffset = {};
    10485        this._tabs = $(this).tabs({

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures.js

    r106739 r116599  
    3232
    3333ui.failures.Builder = base.extends('a', {
    34     init: function(builderName)
     34    init: function(builderName, failures)
    3535    {
    3636        var configuration = config.kBuilders[builderName];
    37         if (configuration.version)
    38             this._addSpan('version', configuration.version);
    39         if (configuration.is64bit)
    40             this._addSpan('architecture', '64-bit');
    41         this._configuration = configuration;
     37        if (configuration) {
     38            if (configuration.version)
     39                this._addSpan('version', configuration.version);
     40            if (configuration.is64bit)
     41                this._addSpan('architecture', '64-bit');
     42            this._configuration = configuration;
     43        } else
     44            this._addSpan('version', builderName);
     45
    4246        this.className = 'failing-builder';
    4347        this.target = '_blank';
    4448        this.href = ui.displayURLForBuilder(builderName);
     49        if (failures)
     50            this._addSpan('failures', ' ' + failures.join(', '));
    4551    },
    4652    _addSpan: function(className, text)
    … …  
    5258    equals: function(configuration)
    5359    {
    54         return this._configuration.is64bit == configuration.is64bit && this._configuration.version == configuration.version;
     60        return this._configuration && this._configuration.is64bit == configuration.is64bit && this._configuration.version == configuration.version;
    5561    }
    5662});

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures_unittests.js

    r112355 r116599  
    2828module('ui.failures');
    2929
    30 test('Builder', 7, function() {
    31     raises(function() {
    32         new ui.failures.Builder();
    33     });
    34 
     30test('Builder', 6, function() {
    3531    var configuration;
    36     configuration = new ui.failures.Builder("Webkit Linux");
     32    configuration = new ui.failures.Builder("Webkit Linux", ["update", "webkit_tests"]);
    3733    deepEqual(Object.getOwnPropertyNames(configuration.__proto__).sort(), [
    3834        '_addSpan',
    … …  
    4036        'init',
    4137    ]);
    42     equal(configuration.outerHTML, '<a class="failing-builder" target="_blank" href="http://build.chromium.org/p/chromium.webkit/waterfall?builder=Webkit+Linux"><span class="version">lucid</span><span class="architecture">64-bit</span></a>');
     38    equal(configuration.outerHTML, '<a class="failing-builder" target="_blank" href="http://build.chromium.org/p/chromium.webkit/waterfall?builder=Webkit+Linux"><span class="version">lucid</span><span class="architecture">64-bit</span><span class="failures"> update, webkit_tests</span></a>');
    4339    configuration = new ui.failures.Builder("Webkit Win");
    4440    equal(configuration.outerHTML, '<a class="failing-builder" target="_blank" href="http://build.chromium.org/p/chromium.webkit/waterfall?builder=Webkit+Win"><span class="version">xp</span></a>');

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications.js

    r106739 r116599  
    244244        this._problem.insertBefore(document.createTextNode(message + ':'), this._problem.firstChild);
    245245    },
    246     setFailingBuilders: function(builderNameList)
    247     {
    248         $(this._effects).empty().append(builderNameList.map(function(builderName) {
     246    setFailingBuilders: function(failuresList)
     247    {
     248        $(this._effects).empty().append(Object.keys(failuresList).map(function(builderName) {
    249249            var effect = document.createElement('li');
    250250            effect.className = 'builder';
    251             effect.appendChild(new ui.failures.Builder(builderName));
     251            effect.appendChild(new ui.failures.Builder(builderName, failuresList[builderName]));
    252252            return effect;
    253253        }));

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications_unittests.js

    r111228 r116599  
    319319test('BuildersFailing', 1, function() {
    320320    var builderFailing = new ui.notifications.BuildersFailing('Disasterifying');
    321     builderFailing.setFailingBuilders(['Webkit Linux', 'Webkit Vista']);
     321    builderFailing.setFailingBuilders({'Webkit Linux': ['compile'], 'Webkit Vista': ['webkit_tests', 'update']);
    322322    equal(builderFailing.innerHTML,
    323323        '<div class="how">' +

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui_unittests.js

    r116483 r116599  
    6262        '<div id="unexpected" class="ui-tabs-panel ui-widget-content ui-corner-bottom"></div>' +
    6363        '<div id="expected" class="ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide"></div>' +
    64         '<div id="results" class="ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide"></div>' +
    65         '<div id="chromium-gtests"><span id="gtest-label">Chromium gtest bots:</span>' +
    66             '<iframe scrolling="no" src="http://build.chromium.org/p/chromium.webkit/horizontal_one_box_per_builder?' +
    67                 'builder=Win%20Builder' +
    68                 '&amp;builder=Win%20Reliability%20Builder' +
    69                 '&amp;builder=Vista%20Tests' +
    70                 '&amp;builder=Win%20Reliability' +
    71                 '&amp;builder=Win+(dbg)' +
    72                 '&amp;builder=Win%20Shared%20Builder%20%28dbg%29' +
    73                 '&amp;builder=Mac10.6%20Tests' +
    74                 '&amp;builder=Mac+Builder+(dbg)' +
    75                 '&amp;builder=Linux%20Tests' +
    76                 '&amp;builder=Linux%20Valgrind' +
    77                 '&amp;builder=Vista%20Perf' +
    78                 '&amp;builder=Linux%20Perf' +
    79                 '&amp;builder=Mac10.6%20Perf"></iframe>' +
    80         '</div>');
     64        '<div id="results" class="ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide"></div>');
    8165
    8266    onebar.select('expected');

  • trunk/Tools/ChangeLog

    r116595 r116599  
     12012-05-09  Ojan Vafai  <ojan@chromium.org>
     2
     3        Change "losing test coverage" to cover all non-layout test problems in garden-o-matic
     4        https://bugs.webkit.org/show_bug.cgi?id=86043
     5
     6        Reviewed by Adam Barth.
     7
     8        -Get rid of the gtest iframe.
     9        -Cover all non-layout test problems. Before we wouldn't catch compile failures
     10        or gtest failures since they were not on the webkit testing bots.
     11        -Include the list of failing steps next to the builder name. This is a lot less
     12        cluttered than you'd expect and already helped me find a compile failure early.
     13
     14        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders.js:
     15        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders_unittests.js:
     16        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/config.js:
     17        Delete unused map.
     18        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/controllers.js:
     19        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/garden-o-matic.js:
     20        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui.js:
     21        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures.js:
     22        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures_unittests.js:
     23        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications.js:
     24        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications_unittests.js:
     25        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui_unittests.js:
     26
    1272012-05-09  Jessie Berlin  <jberlin@apple.com>
    228
Note: See TracChangeset for help on using the changeset viewer.