Changeset 106784 in webkit

Timestamp:

Feb 6, 2012 1:07:20 AM (12 years ago)

Author:

sergio@webkit.org

Message:

Incorrect statistics shown when running run-webkit-tests with --repeat-each or --iterations
​https://bugs.webkit.org/show_bug.cgi?id=77672

Reviewed by Dirk Pranke.

Test repetitions must be taken into account when working out
the statistics shown by run-webkit-tests.

• Scripts/webkitpy/layout_tests/controllers/manager.py:

(Manager.prepare_lists_and_print_output):
(Manager._print_result_summary):

• Scripts/webkitpy/layout_tests/models/result_summary.py:

(ResultSummary.init):
(ResultSummary.add):

• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

(MainTest.test_repeat_each_iterations_num_tests):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/controllers/manager.py (modified) (3 diffs)
• Scripts/webkitpy/layout_tests/models/result_summary.py (modified) (3 diffs)
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (modified) (1 diff)

trunk/Tools/ChangeLog

@@ +1 @@
+2012-02-06  Sergio Villar Senin  <svillar@igalia.com>
+
+        Incorrect statistics shown when running run-webkit-tests with --repeat-each or --iterations
+        https://bugs.webkit.org/show_bug.cgi?id=77672
+
+        Reviewed by Dirk Pranke.
+
+        Test repetitions must be taken into account when working out
+        the statistics shown by run-webkit-tests.
+
+        * Scripts/webkitpy/layout_tests/controllers/manager.py:
+        (Manager.prepare_lists_and_print_output):
+        (Manager._print_result_summary):
+        * Scripts/webkitpy/layout_tests/models/result_summary.py:
+        (ResultSummary.__init__):
+        (ResultSummary.add):
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+        (MainTest.test_repeat_each_iterations_num_tests):
+
 2012-02-05  Dan Bernstein  <mitz@apple.com>
 

trunk/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@ -504 +504 @@
             self._test_files_list = self._test_files_list * self._options.iterations
 
-        result_summary = ResultSummary(self._expectations, self._test_files | skipped)
+        iterations =  \
+            (self._options.repeat_each if self._options.repeat_each else 1) * \
+            (self._options.iterations if self._options.iterations else 1)
+        result_summary = ResultSummary(self._expectations, self._test_files | skipped, iterations)
         self._print_expected_results_of_type(result_summary, test_expectations.PASS, "passes")
         self._print_expected_results_of_type(result_summary, test_expectations.FAIL, "failures")
@@ -519 +522 @@
                 result = test_results.TestResult(test)
                 result.type = test_expectations.SKIP
-                result_summary.add(result, expected=True)
+                iterations =  \
+                    (self._options.repeat_each if self._options.repeat_each else 1) * \
+                    (self._options.iterations if self._options.iterations else 1)
+                for iteration in range(iterations):
+                    result_summary.add(result, expected=True)
         self._printer.print_expected('')
 
@@ -1321 +1328 @@
           result_summary: information to log
         """
-        failed = len(result_summary.failures)
-        skipped = len(
-            result_summary.tests_by_expectation[test_expectations.SKIP])
+        failed = result_summary.total_failures
+        skipped = result_summary.total_tests_by_expectation[test_expectations.SKIP]
         total = result_summary.total
         passed = total - failed - skipped

trunk/Tools/Scripts/webkitpy/layout_tests/models/result_summary.py

@@ -32 +32 @@
 
 class ResultSummary(object):
-    def __init__(self, expectations, test_files):
-        self.total = len(test_files)
+    def __init__(self, expectations, test_files, iterations=1):
+        self.total = len(test_files) * iterations
         self.remaining = self.total
         self.expectations = expectations
@@ -41 +41 @@
         self.unexpected_crashes = 0
         self.unexpected_timeouts = 0
+        self.total_tests_by_expectation = {}
         self.tests_by_expectation = {}
         self.tests_by_timeline = {}
@@ -46 +47 @@
         self.unexpected_results = {}
         self.failures = {}
+        self.total_failures = 0
+        self.total_tests_by_expectation[SKIP] = 0
         self.tests_by_expectation[SKIP] = set()
         for expectation in TestExpectations.EXPECTATIONS.values():
             self.tests_by_expectation[expectation] = set()
+            self.total_tests_by_expectation[expectation] = 0
         for timeline in TestExpectations.TIMELINES.values():
             self.tests_by_timeline[timeline] = expectations.get_tests_with_timeline(timeline)
 
     def add(self, test_result, expected):
+        self.total_tests_by_expectation[test_result.type] += 1
         self.tests_by_expectation[test_result.type].add(test_result.test_name)
         self.results[test_result.test_name] = test_result
         self.remaining -= 1
         if len(test_result.failures):
+            self.total_failures += 1
             self.failures[test_result.test_name] = test_result.failures
         if expected:

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@ -391 +391 @@
         self.assertEquals(tests_run, ['passes/image.html', 'passes/text.html', 'passes/image.html', 'passes/text.html'])
 
+    def test_repeat_each_iterations_num_tests(self):
+        # The total number of tests should be: number_of_tests *
+        # repeat_each * iterations
+        host = MockHost()
+        res, out, err, _ = logging_run(['--iterations', '2',
+                                        '--repeat-each', '4',
+                                        '--print', 'everything',
+                                        'passes/text.html', 'failures/expected/text.html'],
+                                       tests_included=True, host=host, record_results=True)
+        self.assertTrue("=> Results: 8/16 tests passed (50.0%)\n" in out.get())
+        self.assertTrue(err.get()[-2] == "All 16 tests ran as expected.\n")
+
     def test_run_chunk(self):
         # Test that we actually select the right chunk