Changeset 130099 in webkit


Ignore:
Timestamp:
Oct 1, 2012 4:48:17 PM (11 years ago)
Author:
rniwa@webkit.org
Message:

Encapsulate private properties in PerfTestRunner better
https://bugs.webkit.org/show_bug.cgi?id=97833

Reviewed by Ojan Vafai.

PerformanceTests:

This patch moves "private" methods and properties of PerfTestRunner into a closure so that they're
inaccssible from outside. Also catch exceptions from test.run, test.done, and other runner code
to ensure we call notifyDone() even if we broke tests. Otherwise DRT will timeout and we end up
waiting for 10 minutes per each broken test on bots.

  • resources/runner.js:

(PerfTestRunner.gc):
(logInDocument): Extracted from PerfTestRunner.log.
(PerfTestRunner.log): Moved.
(logFatalError): Added.
(start): Renamed from PerfTestRunner._start.
(scheduleNextRun): Extracted from PerfTestRunner._runLoop. Also catch any exceptions that happen
in the runner and ignoreWarmUpAndLog so that we don't end up timing out. We call logFatalError in
such cases, which in turn ensures notifyDone() is called.
(ignoreWarmUpAndLog): Renamed from PerfTestRunner._ignoreWarmUpAndLog.
(finish): Extracted from PerfTestRunner._runLoop.
(PerfTestRunner.measureTime): Moved. The initialization of runCount is moved into start().
(measureTimeOnce): Renamed from PerfTestRunner._measureTimeOnce.
(PerfTestRunner.runPerSecond): Moved. Ditto about runCount.
(measureRunsPerSecondOnce): Renamed from PerfTestRunner._measureRunsPerSecondOnce.
(callRunAndMeasureTime): Renamed from PerfTestRunner._perSecondRunnerIterator.

LayoutTests:

Override PerfTestRunner.now instead of PerfTestRunner._perSecondRunnerIterator since the latter
is no longer exposed.

  • fast/harness/perftests/runs-per-second-iterations-expected.txt:
  • fast/harness/perftests/runs-per-second-iterations.html: Increase the runtime of the last 4 runs

since test.timeToRun is no longer supported by PerfTestRunner.

  • fast/harness/perftests/runs-per-second-log-expected.txt:
  • fast/harness/perftests/runs-per-second-log.html: Avoid use numbers that contain primes other

than 2 and 5 as runs because they cause rouding errors in PerfTestRunner.measureRunsPerSecondOnce
and make the output dependent on the underlying floating number implementation.

Location:
trunk
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • trunk/LayoutTests/ChangeLog

    r130098 r130099  
     12012-09-28  Ryosuke Niwa  <rniwa@webkit.org>
     2
     3        Encapsulate private properties in PerfTestRunner better
     4        https://bugs.webkit.org/show_bug.cgi?id=97833
     5
     6        Reviewed by Ojan Vafai.
     7
     8        Override PerfTestRunner.now instead of PerfTestRunner._perSecondRunnerIterator since the latter
     9        is no longer exposed.
     10
     11        * fast/harness/perftests/runs-per-second-iterations-expected.txt:
     12        * fast/harness/perftests/runs-per-second-iterations.html: Increase the runtime of the last 4 runs
     13        since test.timeToRun is no longer supported by PerfTestRunner.
     14        * fast/harness/perftests/runs-per-second-log-expected.txt:
     15        * fast/harness/perftests/runs-per-second-log.html: Avoid use numbers that contain primes other
     16        than 2 and 5 as runs because they cause rouding errors in PerfTestRunner.measureRunsPerSecondOnce
     17        and make the output dependent on the underlying floating number implementation.
     18
    1192012-10-01  Roger Fong  <roger_fong@apple.com>
    220
  • trunk/LayoutTests/fast/harness/perftests/runs-per-second-iterations-expected.txt

    r116916 r130099  
    33On success, you will see a series of "PASS" messages, followed by "TEST COMPLETE".
    44
    5 Returning times: [0, 10, 30, 60, 100, 100, 100, 100, 500]
     5Returning times: [0, 10, 30, 60, 200, 200, 200, 200, 750]
    66PASS callsInIterations[0] is 1
    77PASS callsInIterations[1] is 10
  • trunk/LayoutTests/fast/harness/perftests/runs-per-second-iterations.html

    r116916 r130099  
    1111
    1212var iteration = 0;
    13 
     13var currentTime = 100;
    1414var callsInIterations = [];
    15 var timesInIterations = [0, 10, 30, 60, 100, 100, 100, 100, 500];
     15var timesInIterations = [0, 10, 30, 60, 200, 200, 200, 200, 750];
    1616var logLines = [];
    1717
    18 PerfTestRunner.log = function (line) { logLines.push(line); }
    19 PerfTestRunner._perSecondRunnerIterator = function (callsPerIteration) {
    20     callsInIterations[iteration] = callsPerIteration;
    21     return timesInIterations[iteration++];
     18PerfTestRunner.now = function () {
     19    iteration++;
     20    if (iteration % 2 == 0)
     21        currentTime += timesInIterations.shift();
     22    return currentTime;
    2223}
    2324
     25PerfTestRunner.log = function (line) { logLines.push(line); }
     26
     27var originalTimesInIterations = timesInIterations.slice();
     28
    2429PerfTestRunner.runPerSecond({
    25     run: function () { },
     30    run: function () {
     31        var i = Math.floor(iteration / 2);
     32        if (callsInIterations[i] == undefined)
     33            callsInIterations[i] = 0;
     34        callsInIterations[i]++;
     35    },
    2636    runCount: 1,
    27     timeToRun: 500,
    2837    done: function () {
    29         debug("Returning times: [" + timesInIterations.join(", ") + "]");
     38        debug("Returning times: [" + originalTimesInIterations.join(", ") + "]");
    3039        shouldEvaluateTo("callsInIterations[0]", 1);
    3140        shouldEvaluateTo("callsInIterations[1]", 10);
  • trunk/LayoutTests/fast/harness/perftests/runs-per-second-log-expected.txt

    r129158 r130099  
    22
    33Running 5 times
    4 Ignoring warm-up run (0 runs/s)
    5 1 runs/s
     4Ignoring warm-up run (1 runs/s)
    652 runs/s
    7 3 runs/s
    864 runs/s
    975 runs/s
     88 runs/s
     910 runs/s
    1010
    1111Time:
    12 values 1, 2, 3, 4, 5 runs/s
    13 avg 3 runs/s
    14 median 3 runs/s
    15 stdev 1.41 runs/s
    16 min 1 runs/s
    17 max 5 runs/s
     12values 2, 4, 5, 8, 10 runs/s
     13avg 5.8 runs/s
     14median 5 runs/s
     15stdev 2.86 runs/s
     16min 2 runs/s
     17max 10 runs/s
    1818
  • trunk/LayoutTests/fast/harness/perftests/runs-per-second-log.html

    r125194 r130099  
    1111var runs = 0;
    1212
    13 PerfTestRunner._perSecondRunnerIterator = function (callsPerIteration) {
    14     return 1000 / runs;
     13var iteration = 0;
     14var currentTime = 0;
     15
     16PerfTestRunner.now = function () {
     17    iteration++;
     18    if (iteration % 2 == 0)
     19        currentTime += 1000 / runs;
     20    return currentTime;
    1521}
    1622
     
    1824
    1925var printStatistics = PerfTestRunner.printStatistics;
    20 PerfTestRunner.printStatistics = function (statistics) {
     26PerfTestRunner.printStatistics = function (statistics, title) {
     27    if (statistics.unit == 'bytes')
     28        return;
    2129    statistics.stdev = statistics.stdev.toPrecision(3);
    22     return printStatistics.call(PerfTestRunner, statistics, 'Time:');
     30    return printStatistics.call(PerfTestRunner, statistics, title);
    2331}
    2432
    2533PerfTestRunner.runPerSecond({
    2634    setup: function () {
    27         if (initial)
    28             initial = false;
    29         else
     35        runs++;
     36        // Avoid 3, 6, 7, 9.
     37        if (runs == 6)
     38            runs += 2;
     39        else if (!(runs % 3))
    3040            runs++;
    3141    },
    3242    run: function () { },
    3343    runCount: 5,
    34     timeToRun: 500,
    3544});
    3645
  • trunk/PerformanceTests/ChangeLog

    r130073 r130099  
     12012-10-01  Ryosuke Niwa  <rniwa@webkit.org>
     2
     3        Encapsulate private properties in PerfTestRunner better
     4        https://bugs.webkit.org/show_bug.cgi?id=97833
     5
     6        Reviewed by Ojan Vafai.
     7
     8        This patch moves "private" methods and properties of PerfTestRunner into a closure so that they're
     9        inaccssible from outside. Also catch exceptions from test.run, test.done, and other runner code
     10        to ensure we call notifyDone() even if we broke tests. Otherwise DRT will timeout and we end up
     11        waiting for 10 minutes per each broken test on bots.
     12
     13        * resources/runner.js:
     14        (PerfTestRunner.gc):
     15        (logInDocument): Extracted from PerfTestRunner.log.
     16        (PerfTestRunner.log): Moved.
     17        (logFatalError): Added.
     18        (start): Renamed from PerfTestRunner._start.
     19        (scheduleNextRun): Extracted from PerfTestRunner._runLoop. Also catch any exceptions that happen
     20        in the runner and ignoreWarmUpAndLog so that we don't end up timing out. We call logFatalError in
     21        such cases, which in turn ensures notifyDone() is called.
     22        (ignoreWarmUpAndLog): Renamed from PerfTestRunner._ignoreWarmUpAndLog.
     23        (finish): Extracted from PerfTestRunner._runLoop.
     24        (PerfTestRunner.measureTime): Moved. The initialization of runCount is moved into start().
     25        (measureTimeOnce): Renamed from PerfTestRunner._measureTimeOnce.
     26        (PerfTestRunner.runPerSecond): Moved. Ditto about runCount.
     27        (measureRunsPerSecondOnce): Renamed from PerfTestRunner._measureRunsPerSecondOnce.
     28        (callRunAndMeasureTime): Renamed from PerfTestRunner._perSecondRunnerIterator.
     29
    1302012-10-01  Florin Malita  <fmalita@chromium.org>
    231
  • trunk/PerformanceTests/resources/runner.js

    r129858 r130099  
    3030PerfTestRunner.now = window.performance && window.performance.webkitNow ? function () { return window.performance.webkitNow(); } : Date.now;
    3131
    32 PerfTestRunner.log = function (text) {
    33     if (this._logLines) {
    34         this._logLines.push(text);
    35         return;
    36     }
    37     if (!document.getElementById("log")) {
    38         var pre = document.createElement('pre');
    39         pre.id = 'log';
    40         document.body.appendChild(pre);
    41     }
    42     document.getElementById("log").innerHTML += text + "\n";
    43     window.scrollTo(0, document.body.height);
    44 }
    45 
    4632PerfTestRunner.logInfo = function (text) {
    4733    if (!window.testRunner)
     
    10692}
    10793
    108 PerfTestRunner.storeHeapResults = function() {
    109     if (!window.internals)
    110         return;
    111     this._jsHeapResults.push(this.getUsedJSHeap());
    112     this._mallocHeapResults.push(this.getUsedMallocHeap());
    113 }
    114 
    11594PerfTestRunner.getUsedMallocHeap = function() {
    11695    var stats = window.internals.mallocStatistics();
     
    146125            gcRec(10);
    147126    }
    148 }
    149 
    150 PerfTestRunner._scheduleNextMeasurementOrNotifyDone = function () {
    151     if (this._completedRuns < this._runCount) {
    152         this.gc();
     127};
     128
     129(function () {
     130    var logLines = null;
     131    var completedRuns = -1;
     132    var callsPerIteration = 1;
     133    var currentTest = null;
     134    var results = [];
     135    var jsHeapResults = [];
     136    var mallocHeapResults = [];
     137    var runCount = undefined;
     138
     139    function logInDocument(text) {
     140        if (!document.getElementById("log")) {
     141            var pre = document.createElement('pre');
     142            pre.id = 'log';
     143            document.body.appendChild(pre);
     144        }
     145        document.getElementById("log").innerHTML += text + "\n";
     146        window.scrollTo(0, document.body.height);
     147    }
     148
     149    PerfTestRunner.log = function (text) {
     150        if (logLines)
     151            logLines.push(text);
     152        else
     153            logInDocument(text);
     154    }
     155
     156    function logFatalError(text) {
     157        PerfTestRunner.log(text);
     158        finish();
     159    }
     160
     161    function start(test, runner) {
     162        if (!test) {
     163            logFatalError('Got a bad test object.');
     164            return;
     165        }
     166        currentTest = test;
     167        runCount = test.runCount || 20;
     168        logLines = window.testRunner ? [] : null;
     169        PerfTestRunner.log("Running " + runCount + " times");
     170        scheduleNextRun(runner);
     171    }
     172
     173    function scheduleNextRun(runner) {
     174        PerfTestRunner.gc();
    153175        window.setTimeout(function () {
    154             var measuredValue = PerfTestRunner._runner();
    155             PerfTestRunner.ignoreWarmUpAndLog(measuredValue);
    156             PerfTestRunner._scheduleNextMeasurementOrNotifyDone();
     176            try {
     177                var measuredValue = runner();
     178            } catch (exception) {
     179                logFatalError('Got an exception while running test.run with name=' + exception.name + ', message=' + exception.message);
     180                return;
     181            }
     182
     183            completedRuns++;
     184
     185            try {
     186                ignoreWarmUpAndLog(measuredValue);
     187            } catch (exception) {
     188                logFatalError('Got an exception while logging the result with name=' + exception.name + ', message=' + exception.message);
     189                return;
     190            }
     191
     192            if (completedRuns < runCount)
     193                scheduleNextRun(runner);
     194            else
     195                finish();
    157196        }, 0);
    158     } else {
    159         if (this._description)
    160             this.log("Description: " + this._description);
    161         this.logStatistics(this._results, this.unit, "Time:");
    162         if (this._jsHeapResults.length) {
    163             this.logStatistics(this._jsHeapResults, "bytes", "JS Heap:");
    164             this.logStatistics(this._mallocHeapResults, "bytes", "Malloc:");
    165         }
    166         if (this._logLines) {
    167             var logLines = this._logLines;
    168             this._logLines = null;
    169             var self = this;
    170             logLines.forEach(function(text) { self.log(text); });
    171         }
    172         if (this._test.done)
    173             this._test.done();
     197    }
     198
     199    function ignoreWarmUpAndLog(measuredValue) {
     200        var labeledResult = measuredValue + " " + PerfTestRunner.unit;
     201        if (completedRuns <= 0)
     202            PerfTestRunner.log("Ignoring warm-up run (" + labeledResult + ")");
     203        else {
     204            results.push(measuredValue);
     205            if (window.internals) {
     206                jsHeapResults.push(PerfTestRunner.getUsedJSHeap());
     207                mallocHeapResults.push(PerfTestRunner.getUsedMallocHeap());
     208            }
     209            PerfTestRunner.log(labeledResult);
     210        }
     211    }
     212
     213    function finish() {
     214        try {
     215            if (currentTest.description)
     216                PerfTestRunner.log("Description: " + currentTest.description);
     217            PerfTestRunner.logStatistics(results, PerfTestRunner.unit, "Time:");
     218            if (jsHeapResults.length) {
     219                PerfTestRunner.logStatistics(jsHeapResults, "bytes", "JS Heap:");
     220                PerfTestRunner.logStatistics(mallocHeapResults, "bytes", "Malloc:");
     221            }
     222            if (logLines)
     223                logLines.forEach(logInDocument);
     224            if (currentTest.done)
     225                currentTest.done();
     226        } catch (exception) {
     227            logInDocument('Got an exception while finalizing the test with name=' + exception.name + ', message=' + exception.message);
     228        }
     229
    174230        if (window.testRunner)
    175231            testRunner.notifyDone();
    176232    }
    177 }
    178 
    179 PerfTestRunner._measureTimeOnce = function () {
    180     var start = this.now();
    181     var returnValue = this._test.run.call(window);
    182     var end = this.now();
    183 
    184     if (returnValue - 0 === returnValue) {
    185         if (returnValue <= 0)
    186             this.log("runFunction returned a non-positive value: " + returnValue);
    187         return returnValue;
    188     }
    189 
    190     return end - start;
    191 }
    192 
    193 PerfTestRunner.ignoreWarmUpAndLog = function (result) {
    194     this._completedRuns++;
    195 
    196     var labeledResult = result + " " + this.unit;
    197     if (this._completedRuns <= 0)
    198         this.log("Ignoring warm-up run (" + labeledResult + ")");
    199     else {
    200         this._results.push(result);
    201         this.storeHeapResults();
    202         this.log(labeledResult);
    203     }
    204 }
    205 
    206 PerfTestRunner._start = function(test) {
    207     this._description = test.description || "";
    208     this._completedRuns = -1;
    209     this._callsPerIteration = 1;
    210     this._test = test;
    211 
    212     this._results = [];
    213     this._jsHeapResults = [];
    214     this._mallocHeapResults = [];
    215 
    216     this._logLines = window.testRunner ? [] : null;
    217     this.log("Running " + this._runCount + " times");
    218     this._scheduleNextMeasurementOrNotifyDone();
    219 }
    220 
    221 PerfTestRunner.measureTime = function (test) {
    222     this._runCount = test.runCount || 20;
    223     this.unit = 'ms';
    224     this._runner = this._measureTimeOnce;
    225     this._start(test);
    226 }
    227 
    228 PerfTestRunner.runPerSecond = function (test) {
    229     this._runCount = test.runCount || 20; // Only used by tests in fast/harness/perftests
    230     this.unit = 'runs/s';
    231     this._runner = this._measureRunsPerSecondOnce;
    232     this._start(test);
    233 }
    234 
    235 PerfTestRunner._measureRunsPerSecondOnce = function () {
    236     var timeToRun = this._test.timeToRun || 750;
    237     var totalTime = 0;
    238     var i = 0;
    239     var callsPerIteration = this._callsPerIteration;
    240 
    241     if (this._test.setup)
    242         this._test.setup();
    243 
    244     while (totalTime < timeToRun) {
    245         totalTime += this._perSecondRunnerIterator(callsPerIteration);
    246         i += callsPerIteration;
    247         if (this._completedRuns < 0 && totalTime < 100)
    248             callsPerIteration = Math.max(10, 2 * callsPerIteration);
    249     }
    250     this._callsPerIteration = callsPerIteration;
    251 
    252     return i * 1000 / totalTime;
    253 }
    254 
    255 PerfTestRunner._perSecondRunnerIterator = function (callsPerIteration) {
    256     var startTime = this.now();
    257     for (var i = 0; i < callsPerIteration; i++)
    258         this._test.run.call(window);
    259     return this.now() - startTime;
    260 }
     233
     234    PerfTestRunner.measureTime = function (test) {
     235        PerfTestRunner.unit = 'ms';
     236        start(test, measureTimeOnce);
     237    }
     238
     239    function measureTimeOnce() {
     240        var start = PerfTestRunner.now();
     241        var returnValue = currentTest.run();
     242        var end = PerfTestRunner.now();
     243
     244        if (returnValue - 0 === returnValue) {
     245            if (returnValue <= 0)
     246                PerfTestRunner.log("runFunction returned a non-positive value: " + returnValue);
     247            return returnValue;
     248        }
     249
     250        return end - start;
     251    }
     252
     253    PerfTestRunner.runPerSecond = function (test) {
     254        PerfTestRunner.unit = 'runs/s';
     255        start(test, measureRunsPerSecondOnce);
     256    }
     257
     258    function measureRunsPerSecondOnce() {
     259        var timeToRun = 750;
     260        var totalTime = 0;
     261        var numberOfRuns = 0;
     262
     263        if (currentTest.setup)
     264            currentTest.setup();
     265
     266        while (totalTime < timeToRun) {
     267            totalTime += callRunAndMeasureTime(callsPerIteration);
     268            numberOfRuns += callsPerIteration;
     269            if (completedRuns < 0 && totalTime < 100)
     270                callsPerIteration = Math.max(10, 2 * callsPerIteration);
     271        }
     272
     273        return numberOfRuns * 1000 / totalTime;
     274    }
     275
     276    function callRunAndMeasureTime(callsPerIteration) {
     277        var startTime = PerfTestRunner.now();
     278        for (var i = 0; i < callsPerIteration; i++)
     279            currentTest.run();
     280        return PerfTestRunner.now() - startTime;
     281    }
     282
     283})();
    261284
    262285if (window.testRunner) {
Note: See TracChangeset for help on using the changeset viewer.