Changeset 90652 in webkit


Ignore:
Timestamp:
Jul 8, 2011 12:26:32 PM (13 years ago)
Author:
abarth@webkit.org
Message:

Teach garden-o-matic how to display test results
https://bugs.webkit.org/show_bug.cgi?id=64141

Reviewed by Ojan Vafai.

This patch includes basic infrastructure for probing build.chromium.org
for test results. We only handle text and image tests, not anything
complicated like reftests. Also, we're using the revision/build
independent results store on the server, so we're avoiding that
complication for now.

It's slightly hacky that we need to probe the server to see what kinds
of results exist. A better solution would be to add CORS support to
the server or to use the local server to help.

  • Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
  • Scripts/webkitpy/tool/servers/data/gardeningserver/index.html:
  • Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
  • Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
  • Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
  • Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:
Location:
trunk/Tools
Files:
9 edited

Legend:

Unmodified
Added
Removed
  • trunk/Tools/ChangeLog

    r90651 r90652  
     12011-07-08  Adam Barth  <abarth@webkit.org>
     2
     3        Teach garden-o-matic how to display test results
     4        https://bugs.webkit.org/show_bug.cgi?id=64141
     5
     6        Reviewed by Ojan Vafai.
     7
     8        This patch includes basic infrastructure for probing build.chromium.org
     9        for test results.  We only handle text and image tests, not anything
     10        complicated like reftests.  Also, we're using the revision/build
     11        independent results store on the server, so we're avoiding that
     12        complication for now.
     13
     14        It's slightly hacky that we need to probe the server to see what kinds
     15        of results exist.  A better solution would be to add CORS support to
     16        the server or to use the local server to help.
     17
     18        * Scripts/webkitpy/tool/servers/data/gardeningserver/base.js:
     19        * Scripts/webkitpy/tool/servers/data/gardeningserver/index.html:
     20        * Scripts/webkitpy/tool/servers/data/gardeningserver/main.js:
     21        * Scripts/webkitpy/tool/servers/data/gardeningserver/results.js:
     22        * Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js:
     23        * Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js:
     24
    1252011-07-08  Dirk Pranke  <dpranke@chromium.org>
    226
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base.js

    r90430 r90652  
    22
    33(function(){
     4
     5base.endsWith = function(string, suffix)
     6{
     7    if (suffix.length > string.length)
     8        return false;
     9    var expectedIndex = string.length - suffix.length;
     10    return string.lastIndexOf(suffix) == expectedIndex;
     11};
    412
    513base.joinPath = function(parent, child)
     
    816        return child;
    917    return parent + '/' + child;
     18};
     19
     20base.trimExtension = function(url)
     21{
     22    var index = url.lastIndexOf('.');
     23    if (index == -1)
     24        return url;
     25    return url.substr(0, index);
    1026}
    1127
     
    3046    walkSubtree(tree, '');
    3147    return filteredTree;
    32 }
     48};
     49
     50base.probe = function(url, options)
     51{
     52    var scriptElement = document.createElement('script');
     53    scriptElement.addEventListener('load', function() {
     54        $(scriptElement).detach();
     55        if (options.success)
     56            options.success.call();
     57    }, false);
     58    scriptElement.addEventListener('error', function() {
     59        $(scriptElement).detach();
     60        if (options.error)
     61            options.error.call();
     62    }, false);
     63    scriptElement.src = url;
     64    document.head.appendChild(scriptElement);
     65};
    3366
    3467})();
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/base_unittests.js

    r90430 r90652  
    44    var value = base.joinPath("path/to", "test.html");
    55    equals(value, "path/to/test.html");
     6});
     7
     8test("endsWith", 9, function() {
     9    ok(base.endsWith("xyz", ""));
     10    ok(base.endsWith("xyz", "z"));
     11    ok(base.endsWith("xyz", "yz"));
     12    ok(base.endsWith("xyz", "xyz"));
     13    ok(!base.endsWith("xyz", "wxyz"));
     14    ok(!base.endsWith("xyz", "gwxyz"));
     15    ok(base.endsWith("", ""));
     16    ok(!base.endsWith("", "z"));
     17    ok(!base.endsWith("xyxy", "yx"));
     18});
     19
     20test("trimExtension", 6, function() {
     21    equals(base.trimExtension("xyz"), "xyz");
     22    equals(base.trimExtension("xy.z"), "xy");
     23    equals(base.trimExtension("x.yz"), "x");
     24    equals(base.trimExtension("x.y.z"), "x.y");
     25    equals(base.trimExtension(".xyz"), "");
     26    equals(base.trimExtension(""), "");
    627});
    728
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/index.html

    r90496 r90652  
    4141  color: #888;
    4242}
    43 .failures .builderName, .failures .actual {
     43.builder .builderName, .builder .actual {
    4444  float: left;
    4545  width: 200px;
     46}
     47.results iframe, .results img {
     48  width: 400px;
     49  height: 300px;
    4650}
    4751</style>
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/main.js

    r90496 r90652  
    1818    var faviconURL = 'favicon-' + (hasFailures ? 'red' : 'green') + '.png';
    1919    $('#favicon').attr('href', faviconURL);
    20 };
     20}
    2121
    2222function fetchResults(onsuccess)
    2323{
    2424    results.fetchResultsByBuilder(config.builders, function(resultsByBuilder) {
    25         var unexpectedFailures = ui.resultsByTest(results.unexpectedFailuresByTest(resultsByBuilder));
     25        var unexpectedFailures = ui.summarizeResultsByTest(results.unexpectedFailuresByTest(resultsByBuilder));
    2626        $('.failures').append(unexpectedFailures);
    2727        onsuccess();
     
    3030}
    3131
     32function expandResults()
     33{
     34    var resultsSummary = $(this);
     35    var testName = $('.testName', resultsSummary).text();
     36    $('.builderName', resultsSummary).each(function() {
     37        var builderName = $(this).text();
     38        results.fetchResultsURLs(builderName, testName, function(resultURLs) {
     39            resultsSummary.append(ui.results(resultURLs));
     40        });
     41    });
     42}
     43
    3244$('.hide').live('click', hide);
    3345$('.quit').live('click', quit);
     46$('.results-summary .test').live('click', expandResults);
    3447
    3548$(document).ready(function() {
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results.js

    r90444 r90652  
    77var kResultsName = 'full_results.json';
    88var kMasterName = 'ChromiumWebkit';
     9
     10var kLayoutTestResultsServer = 'http://build.chromium.org/f/chromium/layout_test_results/';
     11var kLayoutTestResultsPath = '/results/layout-test-results/';
     12
     13var kPossibleSuffixList = [
     14    '-expected.png',
     15    '-actual.png',
     16    '-diff.png',
     17    '-expected.txt',
     18    '-actual.txt',
     19    '-diff.txt',
     20    // FIXME: Add support for these result types.
     21    // '-wdiff.html',
     22    // '-pretty-diff.html',
     23    // '-expected.html',
     24    // '-expected-mismatch.html',
     25    // '-expected.wav',
     26    // '-actual.wav',
     27    // ... and possibly more.
     28];
    929
    1030var PASS = 'PASS';
     
    1737var kFailingResults = [TIMEOUT, TEXT, CRASH, IMAGE, IMAGE_TEXT];
    1838
     39// Kinds of results.
     40results.kActualKind = 'actual';
     41results.kExpectedKind = 'expected';
     42results.kDiffKind = 'diff';
     43results.kUnknownKind = 'unknown';
     44
     45// Types of tests.
     46results.kImageType = 'image'
     47results.kTextType = 'text'
     48// FIXME: There are more types of tests.
     49
    1950function isFailure(result)
    2051{
     
    90121};
    91122
    92 function resultsURL(builderName, name)
     123function resultsDirectoryForBuilder(builderName)
     124{
     125    return builderName.replace(/[ .()]/g, '_');
     126}
     127
     128function resultsDirectoryURL(builderName)
     129{
     130    return kLayoutTestResultsServer + resultsDirectoryForBuilder(builderName) + kLayoutTestResultsPath;
     131}
     132
     133results.resultKind = function(url)
     134{
     135    if (/-actual\.[a-z]+$/.test(url))
     136        return results.kActualKind;
     137    else if (/-expected\.[a-z]+$/.test(url))
     138        return results.kExpectedKind;
     139    else if (/diff\.[a-z]+$/.test(url))
     140        return results.kDiffKind;
     141    return results.kUnknownKind;
     142}
     143
     144results.resultType = function(url)
     145{
     146    if (/\.png$/.test(url))
     147        return results.kImageType;
     148    return results.kTextType;
     149}
     150
     151function sortResultURLsBySuffix(urls)
     152{
     153    var sortedURLs = [];
     154    $.each(kPossibleSuffixList, function(i, suffix) {
     155        $.each(urls, function(j, url) {
     156            if (!base.endsWith(url, suffix))
     157                return;
     158            sortedURLs.push(url);
     159        });
     160    });
     161    if (sortedURLs.length != urls.length)
     162        throw "sortResultURLsBySuffix failed to return the same number of URLs."
     163    return sortedURLs;
     164}
     165
     166results.fetchResultsURLs = function(builderName, testName, callback)
     167{
     168    var stem = resultsDirectoryURL(builderName);
     169    var testNameStem = base.trimExtension(testName);
     170
     171    var resultURLs = [];
     172    var requestsInFlight = kPossibleSuffixList.length;
     173
     174    function checkComplete()
     175    {
     176        if (--requestsInFlight == 0)
     177            callback(sortResultURLsBySuffix(resultURLs));
     178    }
     179
     180    $.each(kPossibleSuffixList, function(index, suffix) {
     181        var url = stem + testNameStem + suffix;
     182        base.probe(url, {
     183            success: function() {
     184                resultURLs.push(url);
     185                checkComplete();
     186            },
     187            error: checkComplete,
     188        });
     189    });
     190};
     191
     192function resultsSummaryURL(builderName, name)
    93193{
    94194    return kTestResultsServer + 'testfile' +
     
    102202{
    103203    $.ajax({
    104         url: resultsURL(builderName, kResultsName),
     204        url: resultsSummaryURL(builderName, kResultsName),
    105205        dataType: 'jsonp',
    106206        success: function(data) {
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/results_unittests.js

    r90442 r90652  
    6363    });
    6464});
     65
     66test("resultKind", 12, function() {
     67    equals(results.resultKind("http://example.com/foo-actual.txt"), "actual");
     68    equals(results.resultKind("http://example.com/foo-expected.txt"), "expected");
     69    equals(results.resultKind("http://example.com/foo-diff.txt"), "diff");
     70    equals(results.resultKind("http://example.com/foo.bar-actual.txt"), "actual");
     71    equals(results.resultKind("http://example.com/foo.bar-expected.txt"), "expected");
     72    equals(results.resultKind("http://example.com/foo.bar-diff.txt"), "diff");
     73    equals(results.resultKind("http://example.com/foo-actual.png"), "actual");
     74    equals(results.resultKind("http://example.com/foo-expected.png"), "expected");
     75    equals(results.resultKind("http://example.com/foo-diff.png"), "diff");
     76    equals(results.resultKind("http://example.com/foo-pretty-diff.html"), "diff");
     77    equals(results.resultKind("http://example.com/foo-wdiff.html"), "diff");
     78    equals(results.resultKind("http://example.com/foo-xyz.html"), "unknown");
     79});
     80
     81test("resultType", 12, function() {
     82    equals(results.resultType("http://example.com/foo-actual.txt"), "text");
     83    equals(results.resultType("http://example.com/foo-expected.txt"), "text");
     84    equals(results.resultType("http://example.com/foo-diff.txt"), "text");
     85    equals(results.resultType("http://example.com/foo.bar-actual.txt"), "text");
     86    equals(results.resultType("http://example.com/foo.bar-expected.txt"), "text");
     87    equals(results.resultType("http://example.com/foo.bar-diff.txt"), "text");
     88    equals(results.resultType("http://example.com/foo-actual.png"), "image");
     89    equals(results.resultType("http://example.com/foo-expected.png"), "image");
     90    equals(results.resultType("http://example.com/foo-diff.png"), "image");
     91    equals(results.resultType("http://example.com/foo-pretty-diff.html"), "text");
     92    equals(results.resultType("http://example.com/foo-wdiff.html"), "text");
     93    equals(results.resultType("http://example.com/foo.xyz"), "text");
     94});
     95
     96test("fetchResultsURLs", 3, function() {
     97    var realBase = window.base;
     98
     99    var pendingCallbacks = {};
     100    window.base = {};
     101    base.probe = function(url, options) {
     102        pendingCallbacks[url] = options;
     103    };
     104    base.endsWith = realBase.endsWith;
     105    base.trimExtension = realBase.trimExtension;
     106
     107    results.fetchResultsURLs("Mock Builder", "userscripts/another-test.html", function(resultURLs) {
     108        deepEqual(resultURLs, [
     109            "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
     110            "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
     111            "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
     112        ]);
     113    });
     114
     115    var probedURLs = [];
     116    for (var url in pendingCallbacks) {
     117        probedURLs.push(url);
     118        if (realBase.endsWith(url, '.txt'))
     119            pendingCallbacks[url].success.call();
     120        else
     121            pendingCallbacks[url].error.call();
     122    }
     123
     124    deepEqual(probedURLs, [
     125        "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.png",
     126        "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.png",
     127        "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.png",
     128        "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-expected.txt",
     129        "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-actual.txt",
     130        "http://build.chromium.org/f/chromium/layout_test_results/Mock_Builder/results/layout-test-results/userscripts/another-test-diff.txt",
     131    ]);
     132
     133    window.base = realBase;
     134    equal(window.base, realBase, "Failed to restore real base!");
     135});
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui.js

    r90441 r90652  
    33(function () {
    44
    5 ui.resultsByTest = function(resultsByTest)
     5ui.summarizeResultsByTest = function(resultsByTest)
    66{
    7     var block = $('<div class="results"></div>');
     7    var block = $('<div class="results-summary"></div>');
    88    $.each(resultsByTest, function(testName, resultNodesByBuilder) {
    99        var testBlock = $('<div class="test"><div class="testName"></div><div class="builders"></div></div>');
     
    2222};
    2323
     24ui.results = function(resultsURLs)
     25{
     26    var block = $('<div class="results"></div>');
     27    $.each(resultsURLs, function(index, resultURL) {
     28        var kind = results.resultKind(resultURL);
     29        var type = results.resultType(resultURL);
     30        var fragment = type == results.kImageType ? '<img>' : '<iframe></iframe>';
     31        block.append($(fragment).attr('src', resultURL).addClass(kind))
     32    });
     33    return block;
     34};
     35
    2436})();
  • trunk/Tools/Scripts/webkitpy/tool/servers/data/gardeningserver/ui_unittests.js

    r90441 r90652  
    2020}
    2121
    22 test("BuilderResults.resultsByTest", 1, function() {
    23     var resultsByTest = ui.resultsByTest(kExampleResultsByTest);
    24     equal(resultsByTest.html(),
     22test("summarizeResultsByTest", 1, function() {
     23    var resultsSummary = ui.summarizeResultsByTest(kExampleResultsByTest);
     24    equal(resultsSummary.html(),
    2525        '<div class="test">' +
    2626            '<div class="testName">scrollbars/custom-scrollbar-with-incomplete-style.html</div>' +
     
    3737        '</div>');
    3838});
     39
     40test("results", 1, function() {
     41    var testResults = ui.results([
     42        'http://example.com/layout-test-results/foo-bar-expected.txt',
     43        'http://example.com/layout-test-results/foo-bar-actual.txt',
     44        'http://example.com/layout-test-results/foo-bar-diff.txt',
     45        'http://example.com/layout-test-results/foo-bar-expected.png',
     46        'http://example.com/layout-test-results/foo-bar-actual.png',
     47        'http://example.com/layout-test-results/foo-bar-diff.png',
     48    ]);
     49    equal(testResults.html(),
     50        '<iframe src="http://example.com/layout-test-results/foo-bar-expected.txt" class="expected"></iframe>' +
     51        '<iframe src="http://example.com/layout-test-results/foo-bar-actual.txt" class="actual"></iframe>' +
     52        '<iframe src="http://example.com/layout-test-results/foo-bar-diff.txt" class="diff"></iframe>' +
     53        '<img src="http://example.com/layout-test-results/foo-bar-expected.png" class="expected">' +
     54        '<img src="http://example.com/layout-test-results/foo-bar-actual.png" class="actual">' +
     55        '<img src="http://example.com/layout-test-results/foo-bar-diff.png" class="diff">');
     56});
Note: See TracChangeset for help on using the changeset viewer.