Changeset 162183 in webkit

Timestamp:

Jan 16, 2014 10:06:36 PM (10 years ago)

Author:

rniwa@webkit.org

Message:

Automate DoYouEvenBench
​https://bugs.webkit.org/show_bug.cgi?id=124497

Reviewed by Geoffrey Garen.

PerformanceTests:

Enable DoYouEvenBench/Full.html on perf bots by default.

Put a space between the time and ms, and fixed a typo in runner.js so that the aggregator name will be reported.

• DoYouEvenBench/Full.html:
• Skipped:
• resources/runner.js:

Tools:

• Scripts/webkitpy/performance_tests/perftest.py:

(PerfTestMetric.init): Added the aggregator name as an argument.
(PerfTestMetric.aggregator): Added.
(PerfTest._metrics_regex): Made the subtest name match non-greedy so that the metric names will be
won't be eagerly parsed as a part of the subtest name. e.g. "Time" and "Total" in "a:Time:Total"
should be parsed as the metric and the aggregator respectively.
(PerfTest._run_with_driver): Pass in the aggregator name.
(PerfTest._ensure_metrics): Ditto. Also split the subtest name by / as required by DoYouEvenBench
which generates subtests of subtests within a single test file.

• Scripts/webkitpy/performance_tests/perftest_unittest.py:

(test_parse_output_with_subtests_and_total): Added.

• Scripts/webkitpy/performance_tests/perftestsrunner.py:

(_generate_results_dict): Add the aggregator name to the JSON when one is available.

• Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py:

(TestWithSubtestsData): Added a sub test with an aggregator and a sub-sub test.

Location:

trunk

Files:

• PerformanceTests/ChangeLog (modified) (1 diff)
• PerformanceTests/DoYouEvenBench/Full.html (modified) (2 diffs)
• PerformanceTests/Skipped (modified) (1 diff)
• PerformanceTests/resources/runner.js (modified) (1 diff)
• Tools/ChangeLog (modified) (1 diff)
• Tools/Scripts/webkitpy/performance_tests/perftest.py (modified) (5 diffs)
• Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py (modified) (1 diff)
• Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py (modified) (1 diff)
• Tools/Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py (modified) (2 diffs)

trunk/PerformanceTests/ChangeLog

@@ +1 @@
+2014-01-16  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Automate DoYouEvenBench
+        https://bugs.webkit.org/show_bug.cgi?id=124497
+
+        Reviewed by Geoffrey Garen.
+
+        Enable DoYouEvenBench/Full.html on perf bots by default.
+
+        Put a space between the time and ms, and fixed a typo in runner.js so that the aggregator name will be reported.
+
+        * DoYouEvenBench/Full.html:
+        * Skipped:
+        * resources/runner.js:
+
 2014-01-15  Manuel Rego Casasnovas  <rego@igalia.com>
 

trunk/PerformanceTests/DoYouEvenBench/Full.html

@@ -18 +18 @@
             values.push(measuredValues.total);
             iterationNumber++;
-            pre.appendChild(document.createTextNode('Iteration ' + iterationNumber + ': ' + measuredValues.total + 'ms\n'));
+            pre.appendChild(document.createTextNode('Iteration ' + iterationNumber + ': ' + measuredValues.total + ' ms\n'));
         },
         didFinishLastIteration: function () {
@@ -24 +24 @@
             for (var i = 0; i < values.length; i++)
                 sum += values[i];
-            pre.appendChild(document.createTextNode('Average: ' + (sum / iterationNumber)  + 'ms\n'));
+            pre.appendChild(document.createTextNode('Average: ' + (sum / iterationNumber)  + ' ms\n'));
             pre.style.paddingTop = 0;
         }

trunk/PerformanceTests/Skipped

@@ -90 +90 @@
 Layout/LineLayoutJapanese.html
 
-# New DOM benchmark is not ready for the prime time yet.
-DoYouEvenBench
+# Don't run the interactive runner. We run Full.html
+DoYouEvenBench/benchmark.html

trunk/PerformanceTests/resources/runner.js

@@ -222 +222 @@
                 PerfTestRunner.log("Description: " + currentTest.description);
             metric = {'fps': 'FrameRate', 'runs/s': 'Runs', 'ms': 'Time'}[PerfTestRunner.unit];
-            var suffix = currentTest.aggregation ? ':' + currentTest.aggregation : '';
+            var suffix = currentTest.aggregator ? ':' + currentTest.aggregator : '';
             PerfTestRunner.logStatistics(results, PerfTestRunner.unit, prefix + ":" + metric + suffix);
             if (jsHeapResults.length) {

trunk/Tools/ChangeLog

@@ +1 @@
+2014-01-16  Ryosuke Niwa  <rniwa@webkit.org>
+
+        Automate DoYouEvenBench
+        https://bugs.webkit.org/show_bug.cgi?id=124497
+
+        Reviewed by Geoffrey Garen.
+
+        * Scripts/webkitpy/performance_tests/perftest.py:
+        (PerfTestMetric.__init__): Added the aggregator name as an argument.
+        (PerfTestMetric.aggregator): Added.
+        (PerfTest._metrics_regex): Made the subtest name match non-greedy so that the metric names will be
+        won't be eagerly parsed as a part of the subtest name. e.g. "Time" and "Total" in "a:Time:Total"
+        should be parsed as the metric and the aggregator respectively.
+        (PerfTest._run_with_driver): Pass in the aggregator name.
+        (PerfTest._ensure_metrics): Ditto. Also split the subtest name by / as required by DoYouEvenBench
+        which generates subtests of subtests within a single test file.
+
+        * Scripts/webkitpy/performance_tests/perftest_unittest.py:
+        (test_parse_output_with_subtests_and_total): Added.
+
+        * Scripts/webkitpy/performance_tests/perftestsrunner.py:
+        (_generate_results_dict): Add the aggregator name to the JSON when one is available.
+
+        * Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py:
+        (TestWithSubtestsData): Added a sub test with an aggregator and a sub-sub test.
+
 2014-01-16  Chris Fleizach  <cfleizach@apple.com>
 

trunk/Tools/Scripts/webkitpy/performance_tests/perftest.py

@@ -51 +51 @@
 
 class PerfTestMetric(object):
-    def __init__(self, path, test_file_name, metric, unit=None, iterations=None):
+    def __init__(self, path, test_file_name, metric, unit=None, aggregator=None, iterations=None):
         # FIXME: Fix runner.js to report correct metric names
         self._iterations = iterations or []
         self._unit = unit or self.metric_to_unit(metric)
+        self._aggregator = aggregator
         self._metric = self.time_unit_to_metric(self._unit) if metric == 'Time' else metric
         self._path = path
@@ -61 +62 @@
     def name(self):
         return self._metric
+
+    def aggregator(self):
+        return self._aggregator
 
     def path(self):
@@ -169 +173 @@
 
     _description_regex = re.compile(r'^Description: (?P<description>.*)$', re.IGNORECASE)
-    _metrics_regex = re.compile(r'^(?P<subtest>[A-Za-z0-9\(\[].+)?:(?P<metric>[A-Z][A-Za-z]+)(:(?P<aggregator>[A-Z][A-Za-z]+))? -> \[(?P<values>(\d+(\.\d+)?)(, \d+(\.\d+)?)+)\] (?P<unit>[a-z/]+)?$')
+    _metrics_regex = re.compile(r'^(?P<subtest>[A-Za-z0-9\(\[].+?)?:(?P<metric>[A-Z][A-Za-z]+)(:(?P<aggregator>[A-Z][A-Za-z]+))? -> \[(?P<values>(\d+(\.\d+)?)(, \d+(\.\d+)?)+)\] (?P<unit>[a-z/]+)?$')
 
     def _run_with_driver(self, driver, time_out_ms):
@@ -189 +193 @@
                 return False
 
-            metric = self._ensure_metrics(metric_match.group('metric'), metric_match.group('subtest'), metric_match.group('unit'))
+            metric = self._ensure_metrics(metric_match.group('metric'), metric_match.group('subtest'), metric_match.group('unit'), metric_match.group('aggregator'))
             metric.append_group(map(lambda value: float(value), metric_match.group('values').split(', ')))
 
         return True
 
-    def _ensure_metrics(self, metric_name, subtest_name='', unit=None):
+    def _ensure_metrics(self, metric_name, subtest_name='', unit=None, aggregator=None):
         try:
             subtest = next(subtest for subtest in self._metrics if subtest['name'] == subtest_name)
@@ -206 +210 @@
             path = self.test_name_without_file_extension().split('/')
             if subtest_name:
-                path += [subtest_name]
-            metric = PerfTestMetric(path, self._test_name, metric_name, unit)
+                path += subtest_name.split('/')
+            metric = PerfTestMetric(path, self._test_name, metric_name, unit, aggregator)
             subtest['metrics'].append(metric)
             return metric

trunk/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py

@@ -212 +212 @@
 """)
 
+    def test_parse_output_with_subtests_and_total(self):
+        output = DriverOutput("""
+:Time:Total -> [2324, 2328, 2345, 2314, 2312] ms
+EmberJS-TodoMVC:Time:Total -> [1462, 1473, 1490, 1465, 1458] ms
+EmberJS-TodoMVC/a:Time -> [1, 2, 3, 4, 5] ms
+BackboneJS-TodoMVC:Time -> [862, 855, 855, 849, 854] ms
+""", image=None, image_hash=None, audio=None)
+        output_capture = OutputCapture()
+        output_capture.capture_output()
+        try:
+            test = PerfTest(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test')
+            test.run_single = lambda driver, path, time_out_ms: output
+            self.assertTrue(test.run(10))
+        finally:
+            actual_stdout, actual_stderr, actual_logs = output_capture.restore_output()
+
+        subtests = test._metrics
+        self.assertEqual(map(lambda test: test['name'], subtests), [None, 'EmberJS-TodoMVC', 'EmberJS-TodoMVC/a', 'BackboneJS-TodoMVC'])
+
+        main_metrics = subtests[0]['metrics']
+        self.assertEqual(map(lambda metric: metric.name(), main_metrics), ['Time'])
+        self.assertEqual(main_metrics[0].aggregator(), 'Total')
+        self.assertEqual(main_metrics[0].path(), ['some-dir', 'some-test'])
+        self.assertEqual(main_metrics[0].flattened_iteration_values(), [2324, 2328, 2345, 2314, 2312] * 4)
+
+        some_test_metrics = subtests[1]['metrics']
+        self.assertEqual(map(lambda metric: metric.name(), some_test_metrics), ['Time'])
+        self.assertEqual(some_test_metrics[0].aggregator(), 'Total')
+        self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'EmberJS-TodoMVC'])
+        self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1462, 1473, 1490, 1465, 1458] * 4)
+
+        some_test_metrics = subtests[2]['metrics']
+        self.assertEqual(map(lambda metric: metric.name(), some_test_metrics), ['Time'])
+        self.assertEqual(some_test_metrics[0].aggregator(), None)
+        self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'EmberJS-TodoMVC', 'a'])
+        self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [1, 2, 3, 4, 5] * 4)
+
+        some_test_metrics = subtests[3]['metrics']
+        self.assertEqual(map(lambda metric: metric.name(), some_test_metrics), ['Time'])
+        self.assertEqual(some_test_metrics[0].aggregator(), None)
+        self.assertEqual(some_test_metrics[0].path(), ['some-dir', 'some-test', 'BackboneJS-TodoMVC'])
+        self.assertEqual(some_test_metrics[0].flattened_iteration_values(), [862, 855, 855, 849, 854] * 4)
+
+        self.assertEqual(actual_stdout, '')
+        self.assertEqual(actual_stderr, '')
+        self.assertEqual(actual_logs, """RESULT some-dir: some-test: Time= 2324.6 ms
+median= 2324.0 ms, stdev= 12.1326007105 ms, min= 2312.0 ms, max= 2345.0 ms
+""")
+
 
 class TestSingleProcessPerfTest(unittest.TestCase):

trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py

@@ -270 +270 @@
                     current_test.setdefault('metrics', {})
                     assert metric.name() not in current_test['metrics']
-                    current_test['metrics'][metric.name()] = {'current': metric.grouped_iteration_values()}
+                    test_results = {'current': metric.grouped_iteration_values()}
+                    if metric.aggregator():
+                        test_results['aggregators'] = [metric.aggregator()]
+                    current_test['metrics'][metric.name()] = test_results
                 else:
                     current_test.setdefault('tests', {})

trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_integrationtest.py

@@ -99 +99 @@
 class TestWithSubtestsData:
     text = """subtest:Time -> [1, 2, 3, 4, 5] ms
+total-test:Time:Total -> [1, 2, 3, 4, 5] ms
+total-test/subsubtest:Time -> [1, 2, 3, 4, 5] ms
 :Time -> [1080, 1120, 1095, 1101, 1104] ms
 """
@@ -114 +116 @@
             'subtest': {
                 'url': 'http://trac.webkit.org/browser/trunk/PerformanceTests/Parser/test-with-subtests.html',
-                'metrics': {'Time': {'current': [[1.0, 2.0, 3.0, 4.0, 5.0]] * 4}}}}}
+                'metrics': {'Time': {'current': [[1.0, 2.0, 3.0, 4.0, 5.0]] * 4}}},
+            'total-test': {
+                'url': 'http://trac.webkit.org/browser/trunk/PerformanceTests/Parser/test-with-subtests.html',
+                'metrics': {'Time': {'current': [[1.0, 2.0, 3.0, 4.0, 5.0]] * 4, "aggregators": ["Total"]}},
+                'tests': {
+                    'subsubtest':
+                        {'url': 'http://trac.webkit.org/browser/trunk/PerformanceTests/Parser/test-with-subtests.html',
+                        'metrics': {'Time': {'current': [[1.0, 2.0, 3.0, 4.0, 5.0]] * 4}}}}}}}