Changeset 246303 in webkit

Timestamp:

Jun 10, 2019 10:05:19 PM (5 years ago)

Author:

Dewei Zhu

Message:

Extend run-benchmark to allow diagnosing before closing browser on test failure.
​https://bugs.webkit.org/show_bug.cgi?id=198729

Reviewed by Ryosuke Niwa.

Add '--diagnose-directory' option to store diagnose information when test failed.

• Scripts/webkitpy/benchmark_runner/benchmark_runner.py:

(BenchmarkRunner.init):

• Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py:

(BrowserDriver.diagnose_test_failure): Add default no-op function to base class.

• Scripts/webkitpy/benchmark_runner/run_benchmark.py: Added '--diagnose-directory' option.

(parse_args):
(run_benchmark_plan):

• Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py:

(WebDriverBenchmarkRunner._run_one_test): Added 'diagnose_test_failure' invocation on test failure.

• Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py:

(WebServerBenchmarkRunner.init):
(WebServerBenchmarkRunner._run_one_test): Added 'diagnose_test_failure' invocation on test failure.

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/benchmark_runner/benchmark_runner.py (modified) (2 diffs)
• Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py (modified) (1 diff)
• Scripts/webkitpy/benchmark_runner/run_benchmark.py (modified) (2 diffs)
• Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py (modified) (1 diff)
• Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py (modified) (2 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2019-06-10  Dewei Zhu  <dewei_zhu@apple.com>
+
+        Extend run-benchmark to allow diagnosing before closing browser on test failure.
+        https://bugs.webkit.org/show_bug.cgi?id=198729
+
+        Reviewed by Ryosuke Niwa.
+
+        Add '--diagnose-directory' option to store diagnose information when test failed.
+
+        * Scripts/webkitpy/benchmark_runner/benchmark_runner.py:
+        (BenchmarkRunner.__init__):
+        * Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py:
+        (BrowserDriver.diagnose_test_failure): Add default no-op function to base class.
+        * Scripts/webkitpy/benchmark_runner/run_benchmark.py: Added '--diagnose-directory' option.
+        (parse_args):
+        (run_benchmark_plan):
+        * Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py:
+        (WebDriverBenchmarkRunner._run_one_test): Added 'diagnose_test_failure' invocation on test failure.
+        * Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py:
+        (WebServerBenchmarkRunner.__init__):
+        (WebServerBenchmarkRunner._run_one_test): Added 'diagnose_test_failure' invocation on test failure.
+
 2019-06-10  Sam Weinig  <weinig@apple.com>
 

trunk/Tools/Scripts/webkitpy/benchmark_runner/benchmark_runner.py

@@ -24 +24 @@
     name = 'benchmark_runner'
 
-    def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None):
+    def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None, diagnose_dir=None):
         try:
             plan_file = self._find_plan_file(plan_file)
@@ -39 +39 @@
                 self._browser_path = browser_path
                 self._build_dir = os.path.abspath(build_dir) if build_dir else None
+                self._diagnose_dir = os.path.abspath(diagnose_dir) if diagnose_dir else None
                 self._output_file = output_file
                 self._scale_unit = scale_unit

trunk/Tools/Scripts/webkitpy/benchmark_runner/browser_driver/browser_driver.py

@@ -43 +43 @@
         pass
 
+    def diagnose_test_failure(self, debug_directory, error):
+        pass
+
     @property
     def webdriver_binary_path(self):

trunk/Tools/Scripts/webkitpy/benchmark_runner/run_benchmark.py

@@ -48 +48 @@
     parser.add_argument('--device-id', default=None, help='Undocumented option for mobile device testing.')
     parser.add_argument('--debug', action='store_true', help='Enable debug logging.')
+    parser.add_argument('--diagnose-directory', dest='diagnose_dir', default=None, help='Directory for storing diagnose information on test failure. It\'s up to browser driver implementation when this option is not specified.')
     parser.add_argument('--no-adjust-unit', dest='scale_unit', action='store_false', help="Don't convert to scientific notation.")
     parser.add_argument('--show-iteration-values', dest='show_iteration_values', action='store_true', help="Show the measured value for each iteration in addition to averages.")
@@ -69 +70 @@
 def run_benchmark_plan(args, plan):
     benchmark_runner_class = benchmark_runner_subclasses[args.driver]
-    runner = benchmark_runner_class(plan, args.local_copy, args.count, args.build_dir, args.output_file, args.platform, args.browser, args.browser_path, args.scale_unit, args.show_iteration_values, args.device_id)
+    runner = benchmark_runner_class(plan, args.local_copy, args.count, args.build_dir, args.output_file, args.platform, args.browser, args.browser_path, args.scale_unit, args.show_iteration_values, args.device_id, args.diagnose_dir)
     runner.execute()
 

trunk/Tools/Scripts/webkitpy/benchmark_runner/webdriver_benchmark_runner.py

@@ -27 +27 @@
             result = WebDriverWait(driver, self._plan['timeout'], poll_frequency=1.0).until(self._get_result)
             driver.quit()
+        except Exception as error:
+            self._browser_driver.diagnose_test_failure(self._diagnose_dir, error)
+            raise error
         finally:
             self._browser_driver.close_browsers()

trunk/Tools/Scripts/webkitpy/benchmark_runner/webserver_benchmark_runner.py

@@ -16 +16 @@
     name = 'webserver'
 
-    def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None):
+    def __init__(self, plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit=True, show_iteration_values=False, device_id=None, diagnose_dir=None):
         self._http_server_driver = HTTPServerDriverFactory.create(platform)
         self._http_server_driver.set_device_id(device_id)
-        super(WebServerBenchmarkRunner, self).__init__(plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit, show_iteration_values, device_id)
+        super(WebServerBenchmarkRunner, self).__init__(plan_file, local_copy, count_override, build_dir, output_file, platform, browser, browser_path, scale_unit, show_iteration_values, device_id, diagnose_dir)
 
     def _get_result(self, test_url):
@@ -34 +34 @@
             with Timeout(self._plan['timeout']):
                 result = self._get_result(url)
+        except Exception as error:
+            self._browser_driver.diagnose_test_failure(self._diagnose_dir, error)
+            raise error
         finally:
             self._browser_driver.close_browsers()