Changeset 90922 in webkit


Ignore:
Timestamp:
Jul 13, 2011 9:30:22 AM (13 years ago)
Author:
Adam Roben
Message:

Make TestFailures show every time a possibly-flaky test failed, but hide it by default

It's useful to be able to see every time a flaky test failed to see whether it failed the
same way every time. But doing so takes a lot of space, so the list of failures is now
collapsed by default and can be revealed using a disclosure triangle.

Fixes <​http://webkit.org/b/64455> TestFailures page doesn't show as much information for
flaky tests as I would like, even though the page is already so long

Reviewed by Dimitri Glazkov.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js:

(FlakyLayoutTestDetector.prototype.allFailures): Replaced flakinessExamples with this
function. Now returns all failures for the given test.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js:

Added. This just contains some simple tests of the FlakyLayoutTestDetector class. We'll add
more over time.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js:

(LayoutTestHistoryAnalyzer.prototype.start): Updated the documentation comment to reflect
that we no longer return passing builds for possibly-flaky tests.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:

(.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list): Make the list of
flakiness examples small, too, since it can get quite long.

(.disclosure-triangle):
(.expanded > .disclosure-triangle):
Simple styles for the disclosure triangle.

(.flakiness-examples-list): Collapse the list by default.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:

(ViewController.prototype._displayBuilder): Pass the total number of builds analyzed to
_domForPossiblyFlakyTests.
(ViewController.prototype._domForPossiblyFlakyTests): Put a disclosure triangle to the left
of each test name, and the number of failures to the right. When the disclosure triangle is
clicked for the first time, we build up the list of failures and expand the element. After
that we just collapse or expand the element on subsequent clicks.

  • BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:

Pulled in new tests.

Location:
trunk/Tools
Files:
1 added
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js

    r90682 r90922  
    7171    },
    7272
    73     flakinessExamples: function(testName) {
    74         if (!(testName in this._tests) || this._tests[testName].state !== this._states.PossiblyFlaky)
     73    allFailures: function(testName) {
     74        if (!(testName in this._tests))
    7575            return null;
    7676
    77         var history = this._tests[testName].history;
    78 
    79         var examples = [];
    80         for (var i = 0; i < history.length - 1; ++i) {
    81             var thisIsPassing = history[i].result.failureType === 'pass';
    82             var nextIsPassing = history[i + 1].result.failureType === 'pass';
    83             if (thisIsPassing === nextIsPassing)
    84                 continue;
    85             var last = examples.last();
    86             if (!last || last.build !== history[i].build)
    87                 examples.push(history[i]);
    88             examples.push(history[i + 1]);
    89         }
    90 
    91         // The list of examples can get quite long. Instead of showing the whole list, we abbreviate
    92         // by replacing the middle items with a separator.
    93         const startAndEndAbbreviatedExamplesCount = 3;
    94         console.assert(startAndEndAbbreviatedExamplesCount > 1);
    95         var abbreviatedExamplesToShow = 2 * startAndEndAbbreviatedExamplesCount;
    96         if (examples.length > abbreviatedExamplesToShow) {
    97             var examplesBeforeSeparator = examples.slice(0, startAndEndAbbreviatedExamplesCount);
    98             var examplesAfterSeparator = examples.slice(-startAndEndAbbreviatedExamplesCount);
    99 
    100             // There's no real use in having two "pass" examples in a row immediately next to the
    101             // separator.
    102             if (examplesBeforeSeparator[examplesBeforeSeparator.length - 1].result.failureType === 'pass' && examplesBeforeSeparator[examplesBeforeSeparator.length - 2].result.failureType === 'pass')
    103                 examplesBeforeSeparator.splice(examplesBeforeSeparator.length - 1, 1);
    104             if (examplesAfterSeparator[0].result.failureType === 'pass' && examplesAfterSeparator[1].result.failureType === 'pass')
    105                 examplesAfterSeparator.splice(0, 1);
    106             examples = examplesBeforeSeparator.concat({ isSeparator: true }, examplesAfterSeparator);
    107         }
    108 
    109         return examples;
     77        return this._tests[testName].history.filter(function(historyItem) { return historyItem.result.failureType !== 'pass' });
    11078    },
    11179

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js

    r90571 r90922  
    5757     *     'possiblyFlaky': {
    5858     *         'fast/workers/worker-test.html': [
    59      *             { 'build': 'r12345 (679)', 'result': 'pass' },
    6059     *             { 'build': 'r12344 (678)', 'result': 'fail' },
    61      *             { 'build': 'r12340 (676)', 'result': 'fail' },
    62      *             { 'build': 'r12338 (675)', 'result': 'pass' },
     60     *             { 'build': 'r12340 (676)', 'result': 'crash' },
    6361     *         ],
    6462     *     },
    … …  
    8785                };
    8886                self._flakinessDetector.possiblyFlakyTests.forEach(function(testName) {
    89                     data.possiblyFlaky[testName] = self._flakinessDetector.flakinessExamples(testName);
     87                    data.possiblyFlaky[testName] = self._flakinessDetector.allFailures(testName);
    9088                });
    9189

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css

    r90682 r90922  
    5353}
    5454
    55 .existing-bugs-list, .suspect-revisions-list {
     55.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list {
    5656    font-size: smaller;
    5757}
    … …  
    6565    font-size: larger;
    6666}
     67
     68.disclosure-triangle {
     69    -webkit-transition: -webkit-transform 0.1s;
     70    -webkit-user-select: none;
     71    cursor: default;
     72    display: inline-block;
     73    padding: 0px 3px;
     74}
     75
     76.expanded > .disclosure-triangle {
     77    -webkit-transform: rotateZ(90deg);
     78}
     79
     80.flakiness-examples-list {
     81    -webkit-transition: height 0.25s;
     82    height: 0px;
     83    overflow: hidden;
     84}

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js

    r90814 r90922  
    7171            var list = document.createElement('ol');
    7272            list.id = 'failure-history';
    73             Object.keys(data.history).forEach(function(buildName, buildIndex, buildNameArray) {
     73
     74            var buildNames = Object.keys(data.history)
     75            buildNames.forEach(function(buildName, buildIndex, buildNameArray) {
    7476                var failingTestNames = Object.keys(data.history[buildName].tests);
    7577                if (!failingTestNames.length)
    … …  
    108110            self._mainContentElement.removeAllChildren();
    109111            self._mainContentElement.appendChild(list);
    110             self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky));
     112            self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky, buildNames.length));
    111113
    112114            if (!stillFetchingData)
    … …  
    399401    },
    400402
    401     _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData) {
     403    _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData, buildCount) {
    402404        var result = document.createDocumentFragment();
    403405        var flakyTests = Object.keys(possiblyFlakyTestData);
    … …  
    417419        flakyList.appendChildren(sorted(flakyTests).map(function(testName) {
    418420            var item = document.createElement('li');
    419             item.appendChild(document.createTextNode(testName));
    420             var historyList = document.createElement('ol');
    421             item.appendChild(historyList);
    422             historyList.appendChildren(possiblyFlakyTestData[testName].map(function(historyItem) {
    423                 var item = document.createElement('li');
    424                 if (historyItem.isSeparator) {
    425                     const verticalEllipsis = '\u22ee';
    426                     item.appendChild(document.createTextNode(verticalEllipsis));
    427                     item.className = 'flakiness-example-separator';
    428                     return item;
     421
     422            var disclosureTriangle = document.createElement('span');
     423            item.appendChild(disclosureTriangle);
     424
     425            disclosureTriangle.className = 'disclosure-triangle';
     426            const blackRightPointingSmallTriangle = '\u25b8';
     427            disclosureTriangle.appendChild(document.createTextNode(blackRightPointingSmallTriangle));
     428
     429            var failures = possiblyFlakyTestData[testName];
     430
     431            item.appendChild(document.createTextNode(testName + ' (failed ' + failures.length + ' out of ' + buildCount + ' times)'));
     432
     433            var failureList = document.createElement('ol');
     434            item.appendChild(failureList);
     435
     436            failureList.className = 'flakiness-examples-list';
     437
     438            disclosureTriangle.addEventListener('click', function() {
     439                item.classList.toggle('expanded');
     440                if (!item.classList.contains('expanded')) {
     441                    failureList.style.height = '';
     442                    return;
    429443                }
    430                 item.appendChild(self._domForBuildName(builder, historyItem.build));
    431                 item.appendChild(document.createTextNode(': '));
    432                 item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
    433                 return item;
    434             }));
     444
     445                if (!failureList.firstChild) {
     446                    failureList.appendChildren(failures.map(function(historyItem) {
     447                        var item = document.createElement('li');
     448                        item.appendChild(self._domForBuildName(builder, historyItem.build));
     449                        item.appendChild(document.createTextNode(': '));
     450                        item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
     451                        return item;
     452                    }));
     453                }
     454
     455                // CSS transitions can't transition to a value of 'auto', so we find out the actual
     456                // value using getComputedStyle and transition to that.
     457                failureList.style.height = 'auto';
     458                failureList.style.height = getComputedStyle(failureList).height;
     459            });
     460
    435461            return item;
    436462        }));

  • trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html

    r90814 r90922  
    2020<script src="Builder.js"></script>
    2121<script src="Builder_unittests.js"></script>
     22<script src="FlakyLayoutTestDetector.js"></script>
     23<script src="FlakyLayoutTestDetector_unittests.js"></script>
    2224<script src="NewBugForm.js"></script>
    2325<script src="NewBugForm_unittests.js"></script>

  • trunk/Tools/ChangeLog

    r90921 r90922  
     12011-07-13  Adam Roben  <aroben@apple.com>
     2
     3        Make TestFailures show every time a possibly-flaky test failed, but hide it by default
     4
     5        It's useful to be able to see every time a flaky test failed to see whether it failed the
     6        same way every time. But doing so takes a lot of space, so the list of failures is now
     7        collapsed by default and can be revealed using a disclosure triangle.
     8
     9        Fixes <http://webkit.org/b/64455> TestFailures page doesn't show as much information for
     10        flaky tests as I would like, even though the page is already so long
     11
     12        Reviewed by Dimitri Glazkov.
     13
     14        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js:
     15        (FlakyLayoutTestDetector.prototype.allFailures): Replaced flakinessExamples with this
     16        function. Now returns all failures for the given test.
     17
     18        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js:
     19        Added. This just contains some simple tests of the FlakyLayoutTestDetector class. We'll add
     20        more over time.
     21
     22        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js:
     23        (LayoutTestHistoryAnalyzer.prototype.start): Updated the documentation comment to reflect
     24        that we no longer return passing builds for possibly-flaky tests.
     25
     26        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
     27        (.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list): Make the list of
     28        flakiness examples small, too, since it can get quite long.
     29
     30        (.disclosure-triangle):
     31        (.expanded > .disclosure-triangle):
     32        Simple styles for the disclosure triangle.
     33
     34        (.flakiness-examples-list): Collapse the list by default.
     35
     36        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
     37        (ViewController.prototype._displayBuilder): Pass the total number of builds analyzed to
     38        _domForPossiblyFlakyTests.
     39        (ViewController.prototype._domForPossiblyFlakyTests): Put a disclosure triangle to the left
     40        of each test name, and the number of failures to the right. When the disclosure triangle is
     41        clicked for the first time, we build up the list of failures and expand the element. After
     42        that we just collapse or expand the element on subsequent clicks.
     43
     44        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
     45        Pulled in new tests.
     46
    1472011-07-13  Adam Roben  <aroben@apple.com>
    248       
Note: See TracChangeset for help on using the changeset viewer.