Changeset 112829 in webkit

Timestamp:

Apr 1, 2012 9:48:35 PM (12 years ago)

Author:

rniwa@webkit.org

Message:

perf-o-matic should store test results' units
​https://bugs.webkit.org/show_bug.cgi?id=82852

Reviewed by Kentaro Hara.

.:

• Websites/webkit-perf.appspot.com/models.py:

(Test):
(Test.update_or_insert): Added "unit" to the argument list.
(Test.update_or_insert.execute): Store the unit.
(ReportLog.results_are_well_formed): Moved from ReportHandler.
(ReportLog.results_are_well_formed._is_float_convertible): Ditto.

• Websites/webkit-perf.appspot.com/models_unittest.py:

(TestModelTests.test_update_or_insert): Added a test case for "unit" argument.
(TestModelTests.test_update_or_insert_to_update): Ditto.
(ReportLogTests.test_results_are_well_formed): Added.
(ReportLogTests.test_results_are_well_formed.assert_results_are_well_formed): Added.

• Websites/webkit-perf.appspot.com/report_handler.py:

(ReportHandler.post): Calls ReportLog.results_are_well_formed.

• Websites/webkit-perf.appspot.com/report_process_handler.py:

(ReportProcessHandler.post): Passes resultsunit? to Test.update_or_insert.

Tools:

Include units in the results JSON.

• Scripts/webkitpy/performance_tests/perftestsrunner.py:

(PerfTestsRunner._process_chromium_style_test_result):
(PerfTestsRunner._process_parser_test_result):

Location:

trunk

Files:

• ChangeLog (modified) (1 diff)
• Tools/ChangeLog (modified) (1 diff)
• Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py (modified) (3 diffs)
• Websites/webkit-perf.appspot.com/models.py (modified) (3 diffs)
• Websites/webkit-perf.appspot.com/models_unittest.py (modified) (3 diffs)
• Websites/webkit-perf.appspot.com/report_handler.py (modified) (2 diffs)
• Websites/webkit-perf.appspot.com/report_process_handler.py (modified) (1 diff)

trunk/ChangeLog

@@ +1 @@
+2012-04-01  Ryosuke Niwa  <rniwa@webkit.org>
+
+        perf-o-matic should store test results' units
+        https://bugs.webkit.org/show_bug.cgi?id=82852
+
+        Reviewed by Kentaro Hara.
+
+        * Websites/webkit-perf.appspot.com/models.py:
+        (Test):
+        (Test.update_or_insert): Added "unit" to the argument list.
+        (Test.update_or_insert.execute): Store the unit.
+        (ReportLog.results_are_well_formed): Moved from ReportHandler.
+        (ReportLog.results_are_well_formed._is_float_convertible): Ditto.
+        * Websites/webkit-perf.appspot.com/models_unittest.py:
+        (TestModelTests.test_update_or_insert): Added a test case for "unit" argument.
+        (TestModelTests.test_update_or_insert_to_update): Ditto.
+        (ReportLogTests.test_results_are_well_formed): Added.
+        (ReportLogTests.test_results_are_well_formed.assert_results_are_well_formed): Added.
+        * Websites/webkit-perf.appspot.com/report_handler.py:
+        (ReportHandler.post): Calls ReportLog.results_are_well_formed.
+        * Websites/webkit-perf.appspot.com/report_process_handler.py:
+        (ReportProcessHandler.post): Passes results['unit'] to Test.update_or_insert.
+
 2012-04-01  Ryosuke Niwa  <rniwa@webkit.org>
 

trunk/Tools/ChangeLog

@@ +1 @@
+2012-04-01  Ryosuke Niwa  <rniwa@webkit.org>
+
+        perf-o-matic should store test results' units
+        https://bugs.webkit.org/show_bug.cgi?id=82852
+
+        Reviewed by Kentaro Hara.
+
+        Include units in the results JSON.
+
+        * Scripts/webkitpy/performance_tests/perftestsrunner.py:
+        (PerfTestsRunner._process_chromium_style_test_result):
+        (PerfTestsRunner._process_parser_test_result):
+
 2012-04-01  Tony Tseung  <tseung@apple.com>
 

trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py

@@ -251 +251 @@
             resultLine = self._inspector_result_regex.match(line)
             if resultLine:
+                # FIXME: Store the unit
                 self._results[resultLine.group('name').replace(' ', '')] = float(resultLine.group('value'))
                 self._buildbot_output.write("%s\n" % line)
@@ -287 +288 @@
         score_regex = re.compile(r'^(?P<key>' + r'|'.join(keys) + r')\s+(?P<value>[0-9\.]+)\s*(?P<unit>.*)')
         unit = "ms"
+
         for line in re.split('\n', output.text):
             score = score_regex.match(line)
@@ -301 +303 @@
         if test_failed or set(keys) != set(results.keys()):
             return True
+
+        results['unit'] = unit
+
         self._results[filesystem.join(category, test_name).replace('\\', '/')] = results
         self._buildbot_output.write('RESULT %s: %s= %s %s\n' % (category, test_name, results['avg'], unit))

trunk/Websites/webkit-perf.appspot.com/models.py

@@ -149 +149 @@
     branches = db.ListProperty(db.Key)
     platforms = db.ListProperty(db.Key)
+    unit = db.StringProperty()
     hidden = db.BooleanProperty()
 
     @staticmethod
-    def update_or_insert(test_name, branch, platform):
+    def update_or_insert(test_name, branch, platform, unit=None):
         existing_test = [None]
 
@@ -162 +163 @@
                 if platform.key() not in test.platforms:
                     test.platforms.append(platform.key())
+                test.unit = unit
                 test.put()
                 existing_test[0] = test
@@ -252 +254 @@
     def results(self):
         return self.get_value('results')
+
+    def results_are_well_formed(self):
+
+        def _is_float_convertible(value):
+            try:
+                float(value)
+                return True
+            except TypeError:
+                return False
+            except ValueError:
+                return False
+
+        if not isinstance(self.results(), dict):
+            return False
+
+        for testResult in self.results().values():
+            if isinstance(testResult, dict):
+                for key, value in testResult.iteritems():
+                    if key != "unit" and not _is_float_convertible(value):
+                        return False
+                if 'avg' not in testResult:
+                    return False
+                continue
+            if not _is_float_convertible(testResult):
+                return False
+
+        return True
 
     def builder(self):

trunk/Websites/webkit-perf.appspot.com/models_unittest.py

@@ -266 +266 @@
         self.assertEqual(test.branches, [branch.key()])
         self.assertEqual(test.platforms, [platform.key()])
+        self.assertEqual(test.unit, None)
         self.assertOnlyInstance(test)
 
@@ -276 +277 @@
         other_branch = Branch.create_if_possible('other-branch', 'Other Branch')
         other_platform = Platform.create_if_possible('other-platform', 'Other Platform')
-        test = Test.update_or_insert('some-test', other_branch, other_platform)
+        test = Test.update_or_insert('some-test', other_branch, other_platform, 'ms')
         self.assertOnlyInstance(test)
         self.assertEqualUnorderedList(test.branches, [branch.key(), other_branch.key()])
         self.assertEqualUnorderedList(test.platforms, [platform.key(), other_platform.key()])
-
-        test = Test.get(test.key())
-        self.assertEqualUnorderedList(test.branches, [branch.key(), other_branch.key()])
-        self.assertEqualUnorderedList(test.platforms, [platform.key(), other_platform.key()])
+        self.assertEqualUnorderedList(test.unit, 'ms')
 
     def test_merge(self):
@@ -434 +432 @@
         self.assertEqual(log.results(), None)
 
+    def test_results_are_well_formed(self):
+
+        def assert_results_are_well_formed(json, expected):
+            self.assertEqual(self._create_log_with_payload(json).results_are_well_formed(), expected)
+
+        assert_results_are_well_formed('{"results": 123}', False)
+        assert_results_are_well_formed('{"results": {"test": 123}}', True)
+        assert_results_are_well_formed('{"results": {"test": 123, "other-test": 456}}', True)
+        assert_results_are_well_formed('{"results": {"test": 123, "other-test": 456, "bad-test": "hi"}}', False)
+        assert_results_are_well_formed('{"results": {"test": {"avg": 456}}}', True)
+        assert_results_are_well_formed('{"results": {"test": {"avg": 456, "median": "hello"}}}', False)
+        assert_results_are_well_formed('{"results": {"test": {"avg": 456, "median": 789}}}', True)
+        assert_results_are_well_formed('{"results": {"test": {"avg": 456, "unit": "bytes"}}}', True)
+
     def test_builder(self):
         log = self._create_log_with_payload('{"key": "value"}')

trunk/Websites/webkit-perf.appspot.com/report_handler.py

@@ -70 +70 @@
             self._output('Authentication failed')
 
-        if not self._results_are_valid(log):
+        if not log.results_are_well_formed():
             self._output("The payload doesn't contain results or results are malformed")
 
@@ -88 +88 @@
         return False
 
-    def _results_are_valid(self, log):
-
-        def _is_float_convertible(value):
-            try:
-                float(value)
-                return True
-            except TypeError:
-                return False
-            except ValueError:
-                return False
-
-        if not isinstance(log.results(), dict):
-            return False
-
-        for testResult in log.results().values():
-            if isinstance(testResult, dict):
-                for value in testResult.values():
-                    if not _is_float_convertible(value):
-                        return False
-                if 'avg' not in testResult:
-                    return False
-                continue
-            if not _is_float_convertible(testResult):
-                return False
-
-        return True
-
 
 class AdminReportHandler(ReportHandler):

trunk/Websites/webkit-perf.appspot.com/report_process_handler.py

@@ -59 +59 @@
 
         for test_name, result_value in log.results().iteritems():
-            test = Test.update_or_insert(test_name, branch, platform)
+            test = Test.update_or_insert(test_name, branch, platform, result_value.get('unit'))
             result = TestResult.get_or_insert_from_parsed_json(test_name, build, result_value)
             runs = Runs.get_by_objects(branch, platform, test)