Changeset 123985 in webkit
- Timestamp:
- Jul 29, 2012 3:03:35 PM (12 years ago)
- Location:
- trunk
- Files:
-
- 1 added
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/PerformanceTests/ChangeLog
r123935 r123985 1 2012-07-28 Ryosuke Niwa <rniwa@webkit.org> 2 3 run-perf-tests should generate a results page 4 https://bugs.webkit.org/show_bug.cgi?id=92575 5 6 Reviewed by Eric Seidel. 7 8 Add a results page template. This page includes flot and flot.fillbetween. 9 jQuery and the json that contains test results are injected by run-perf-tests. 10 11 * resources/results-template.html: Added. 12 1 13 2012-07-27 Arnaud Renevier <a.renevier@sisa.samsung.com> 2 14 -
trunk/Tools/ChangeLog
r123984 r123985 1 2012-07-28 Ryosuke Niwa <rniwa@webkit.org> 2 3 run-perf-tests should generate a results page 4 https://bugs.webkit.org/show_bug.cgi?id=92575 5 6 Reviewed by Eric Seidel. 7 8 Added the ability to generate a results page that summarizes performance test results from 9 multiple runs of run-perf-tests when --output-json-path is specified and --test-results-server 10 is not specified. We cannot generate a results page when --test-results-server is specified 11 because perf-o-matic cannot parse the new JSON format. 12 13 The new JSON format is simply an array of the old JSON output. This JSON is then merged into 14 results-template.html along with jQuery and copied as a HTML file of the same filename as 15 the output JSON file with a .html extension. 16 17 We merge all scripts and the JSON output into the results page as opposed to including them 18 with the src content attribute to make it standalone so that we may post it on Bugzilla or 19 e-mail it to someone else without breaking it. 20 21 * Scripts/webkitpy/performance_tests/perftestsrunner.py: 22 (PerfTestsRunner.run): 23 (PerfTestsRunner._generate_json): Merge "contents" with the existing JSON file when generating 24 a results page, and generate the results page from results-template.html merged with jQuery and 25 the generated JSON output. 26 * Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py: 27 (_test_run_with_json_output): Extracted from test_run_with_upload_json to be shared with 28 several test cases. 29 (_test_run_with_json_output.mock_upload_json): Ditto. 30 (test_run_with_json_output): Refactored to use _test_run_with_json_output. 31 (test_run_generates_results_page): Added. Tests the new feature. 32 (test_run_with_json_source): Refactored to use _test_run_with_json_output. 33 (test_run_with_multiple_repositories): Ditto. 34 (test_run_with_upload_json): Ditto. 35 1 36 2012-07-29 Rik Cabanier <cabanier@adobe.com> 2 37 -
trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py
r122289 r123985 160 160 branch = self._default_branch if test_results_server else None 161 161 build_number = int(options.build_number) if options.build_number else None 162 162 163 if not self._generate_json(self._timestamp, options.output_json_path, options.source_json_path, 164 not test_results_server, 163 165 branch, options.platform, options.builder_name, build_number) and not unexpected: 164 166 return self._EXIT_CODE_BAD_JSON 167 165 168 if test_results_server and not self._upload_json(test_results_server, options.output_json_path): 166 169 return self._EXIT_CODE_FAILED_UPLOADING … … 168 171 return unexpected 169 172 170 def _generate_json(self, timestamp, output_json_path, source_json_path, branch, platform, builder_name, build_number): 173 def _generate_json(self, timestamp, output_json_path, source_json_path, should_generate_results_page, 174 branch, platform, builder_name, build_number): 175 171 176 contents = {'timestamp': int(timestamp), 'results': self._results} 172 177 for (name, path) in self._port.repository_paths(): … … 194 199 return False 195 200 196 filesystem.write_text_file(output_json_path, json.dumps(contents)) 201 if should_generate_results_page: 202 if filesystem.isfile(output_json_path): 203 existing_contents = json.loads(filesystem.read_text_file(output_json_path)) 204 existing_contents.append(contents) 205 contents = existing_contents 206 else: 207 contents = [contents] 208 209 serialized_contents = json.dumps(contents) 210 filesystem.write_text_file(output_json_path, serialized_contents) 211 212 if should_generate_results_page: 213 jquery_path = filesystem.join(self._port.perf_tests_dir(), 'Dromaeo/resources/dromaeo/web/lib/jquery-1.6.4.js') 214 jquery = filesystem.read_text_file(jquery_path) 215 216 template_path = filesystem.join(self._port.perf_tests_dir(), 'resources/results-template.html') 217 template = filesystem.read_text_file(template_path) 218 219 results_page = template.replace('<?WebKitPerfTestRunnerInsertionPoint?>', 220 '<script>%s</script><script id="json">%s</script>' % (jquery, serialized_contents)) 221 222 filesystem.write_text_file(filesystem.splitext(output_json_path)[0] + '.html', results_page) 223 197 224 return True 198 225 -
trunk/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py
r121571 r123985 226 226 '', ''])) 227 227 228 def test_run_test_set_with_json_output(self): 229 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json']) 230 port.host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True 231 port.host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True 228 def _test_run_with_json_output(self, runner, filesystem, upload_suceeds=True, expected_exit_code=0): 229 filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content') 230 filesystem.write_text_file(runner._base_path + '/Bindings/event-target-wrapper.html', 'some content') 231 232 uploaded = [False] 233 234 def mock_upload_json(hostname, json_path): 235 self.assertEqual(hostname, 'some.host') 236 self.assertEqual(json_path, '/mock-checkout/output.json') 237 uploaded[0] = True 238 return upload_suceeds 239 240 runner._upload_json = mock_upload_json 232 241 runner._timestamp = 123456789 233 242 output_capture = OutputCapture() 234 243 output_capture.capture_output() 235 244 try: 236 self.assertEqual(runner.run(), 0)245 self.assertEqual(runner.run(), expected_exit_code) 237 246 finally: 238 247 stdout, stderr, logs = output_capture.restore_output() 239 248 240 self.assertEqual(logs, 241 '\n'.join(['Running 2 tests', 242 'Running Bindings/event-target-wrapper.html (1 of 2)', 243 'RESULT Bindings: event-target-wrapper= 1489.05 ms', 244 'median= 1487.0 ms, stdev= 14.46 ms, min= 1471.0 ms, max= 1510.0 ms', 245 '', 246 'Running inspector/pass.html (2 of 2)', 247 'RESULT group_name: test_name= 42 ms', 248 '', ''])) 249 250 self.assertEqual(json.loads(port.host.filesystem.files['/mock-checkout/output.json']), { 251 "timestamp": 123456789, "results": 252 {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, 253 "inspector/pass.html:group_name:test_name": 42}, 254 "webkit-revision": 5678}) 255 256 def test_run_test_set_with_json_source(self): 257 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', '--source-json-path=/mock-checkout/source.json']) 258 port.host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}' 259 port.host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True 260 port.host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True 261 runner._timestamp = 123456789 262 output_capture = OutputCapture() 263 output_capture.capture_output() 264 try: 265 self.assertEqual(runner.run(), 0) 266 finally: 267 stdout, stderr, logs = output_capture.restore_output() 268 269 self.assertEqual(logs, '\n'.join(['Running 2 tests', 249 self.assertEqual(logs, '\n'.join([ 250 'Running 2 tests', 270 251 'Running Bindings/event-target-wrapper.html (1 of 2)', 271 252 'RESULT Bindings: event-target-wrapper= 1489.05 ms', … … 274 255 'Running inspector/pass.html (2 of 2)', 275 256 'RESULT group_name: test_name= 42 ms', 276 '', ''])) 277 257 '', 258 ''])) 259 260 return uploaded[0] 261 262 def test_run_with_json_output(self): 263 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', 264 '--test-results-server=some.host']) 265 self._test_run_with_json_output(runner, port.host.filesystem) 266 self.assertEqual(json.loads(port.host.filesystem.read_text_file('/mock-checkout/output.json')), { 267 "timestamp": 123456789, "results": 268 {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, 269 "inspector/pass.html:group_name:test_name": 42}, 270 "webkit-revision": 5678, "branch": "webkit-trunk"}) 271 272 def test_run_generates_results_page(self): 273 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json']) 274 filesystem = port.host.filesystem 275 print runner._base_path + '/resources/results-template.html' 276 filesystem.write_text_file(runner._base_path + '/resources/results-template.html', 277 'BEGIN<?WebKitPerfTestRunnerInsertionPoint?>END') 278 filesystem.write_text_file(runner._base_path + '/Dromaeo/resources/dromaeo/web/lib/jquery-1.6.4.js', 279 'jquery content') 280 281 self._test_run_with_json_output(runner, filesystem) 282 283 expected_entry = {"timestamp": 123456789, "results": {"Bindings/event-target-wrapper": 284 {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, 285 "inspector/pass.html:group_name:test_name": 42}, "webkit-revision": 5678} 286 287 self.maxDiff = None 288 json_output = port.host.filesystem.read_text_file('/mock-checkout/output.json') 289 self.assertEqual(json.loads(json_output), [expected_entry]) 290 self.assertEqual(filesystem.read_text_file('/mock-checkout/output.html'), 291 'BEGIN<script>jquery content</script><script id="json">' + json_output + '</script>END') 292 293 self._test_run_with_json_output(runner, filesystem) 294 json_output = port.host.filesystem.read_text_file('/mock-checkout/output.json') 295 self.assertEqual(json.loads(json_output), [expected_entry, expected_entry]) 296 self.assertEqual(filesystem.read_text_file('/mock-checkout/output.html'), 297 'BEGIN<script>jquery content</script><script id="json">' + json_output + '</script>END') 298 299 def test_run_with_json_source(self): 300 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', 301 '--source-json-path=/mock-checkout/source.json', '--test-results-server=some.host']) 302 port.host.filesystem.write_text_file('/mock-checkout/source.json', '{"key": "value"}') 303 self._test_run_with_json_output(runner, port.host.filesystem) 278 304 self.assertEqual(json.loads(port.host.filesystem.files['/mock-checkout/output.json']), { 279 305 "timestamp": 123456789, "results": 280 306 {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, 281 307 "inspector/pass.html:group_name:test_name": 42}, 282 "webkit-revision": 5678, 308 "webkit-revision": 5678, "branch": "webkit-trunk", 283 309 "key": "value"}) 284 310 285 def test_run_test_set_with_multiple_repositories(self): 286 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json']) 287 port.host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True 288 runner._timestamp = 123456789 311 def test_run_with_multiple_repositories(self): 312 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', 313 '--test-results-server=some.host']) 289 314 port.repository_paths = lambda: [('webkit', '/mock-checkout'), ('some', '/mock-checkout/some')] 290 self. assertEqual(runner.run(), 0)315 self._test_run_with_json_output(runner, port.host.filesystem) 291 316 self.assertEqual(json.loads(port.host.filesystem.files['/mock-checkout/output.json']), { 292 "timestamp": 123456789, "results": {"inspector/pass.html:group_name:test_name": 42.0}, "webkit-revision": 5678, "some-revision": 5678}) 317 "timestamp": 123456789, "results": 318 {"Bindings/event-target-wrapper": {"max": 1510, "avg": 1489.05, "median": 1487, "min": 1471, "stdev": 14.46, "unit": "ms"}, 319 "inspector/pass.html:group_name:test_name": 42.0}, 320 "webkit-revision": 5678, "some-revision": 5678, "branch": "webkit-trunk"}) 293 321 294 322 def test_run_with_upload_json(self): 295 323 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', 296 324 '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123']) 297 upload_json_is_called = [False] 298 upload_json_returns_true = True 299 300 def mock_upload_json(hostname, json_path): 301 self.assertEqual(hostname, 'some.host') 302 self.assertEqual(json_path, '/mock-checkout/output.json') 303 upload_json_is_called[0] = True 304 return upload_json_returns_true 305 306 runner._upload_json = mock_upload_json 307 port.host.filesystem.files['/mock-checkout/source.json'] = '{"key": "value"}' 308 port.host.filesystem.files[runner._base_path + '/inspector/pass.html'] = True 309 port.host.filesystem.files[runner._base_path + '/Bindings/event-target-wrapper.html'] = True 310 runner._timestamp = 123456789 311 self.assertEqual(runner.run(), 0) 312 self.assertEqual(upload_json_is_called[0], True) 325 326 self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) 313 327 generated_json = json.loads(port.host.filesystem.files['/mock-checkout/output.json']) 314 328 self.assertEqual(generated_json['platform'], 'platform1') 315 329 self.assertEqual(generated_json['builder-name'], 'builder1') 316 330 self.assertEqual(generated_json['build-number'], 123) 317 upload_json_returns_true = False 318 319 runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', 320 '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123']) 321 runner._upload_json = mock_upload_json 322 self.assertEqual(runner.run(), -3) 331 332 self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=False, expected_exit_code=-3) 323 333 324 334 def test_upload_json(self):
Note: See TracChangeset
for help on using the changeset viewer.