Changeset 90086 in webkit

Timestamp:

Jun 29, 2011, 9:16:18 PM (14 years ago)

Author:

dpranke@chromium.org

Message:

2011-06-29 Dirk Pranke <​dpranke@chromium.org>

Reviewed by Adam Barth.

test-webkitpy is spamming lots of messages about updating test expectations
​https://bugs.webkit.org/show_bug.cgi?id=63680

This patch changes the passing_run() function to capture the
output to stdout and stderr and assert that it is empty, and
also specifies --no-new-test-results by default to avoid getting
"generating baseline" messages for the expected-missing results.

Also check that we are logging messages when we generate new
baselines.

Also fix a minor bug in the way test_tolerance() was
checking for whether the tests were passing or not.

• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py (modified) (8 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-06-29  Dirk Pranke  <dpranke@chromium.org>
+
+        Reviewed by Adam Barth.
+
+        test-webkitpy is spamming lots of messages about updating test expectations
+        https://bugs.webkit.org/show_bug.cgi?id=63680
+
+        This patch changes the passing_run() function to capture the
+        output to stdout and stderr and assert that it is empty, and
+        also specifies --no-new-test-results by default to avoid getting
+        "generating baseline" messages for the expected-missing results.
+
+        Also check that we are logging messages when we generate new
+        baselines.
+
+        Also fix a minor bug in the way test_tolerance() was
+        checking for whether the tests were passing or not.
+
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
+
 2011-06-29  Yuta Kitamura  <yutak@chromium.org>
 

trunk/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@ -65 +65 @@
 
 def parse_args(extra_args=None, record_results=False, tests_included=False,
-               print_nothing=True):
+               new_results=False, print_nothing=True):
     extra_args = extra_args or []
     if print_nothing:
@@ -75 +75 @@
     if not record_results:
         args.append('--no-record-results')
+    if not new_results:
+        args.append('--no-new-test-results')
+
     if not '--child-processes' in extra_args and not '--worker-model' in extra_args:
         args.extend(['--worker-model', 'inline'])
@@ -94 +97 @@
         port_obj = port.get(port_name=options.platform, options=options,
                             user=mocktool.MockUser(), filesystem=filesystem)
-    res = run_webkit_tests.run(port_obj, options, parsed_args)
-    return res == 0
-
-
-def logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None):
+    buildbot_output = array_stream.ArrayStream()
+    regular_output = array_stream.ArrayStream()
+    res = run_webkit_tests.run(port_obj, options, parsed_args, buildbot_output=buildbot_output,
+                               regular_output=regular_output)
+    return res == 0 and regular_output.empty() and buildbot_output.empty()
+
+
+def logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None, new_results=False):
     options, parsed_args = parse_args(extra_args=extra_args,
                                       record_results=record_results,
                                       tests_included=tests_included,
-                                      print_nothing=False)
+                                      print_nothing=False, new_results=new_results)
     user = mocktool.MockUser()
     if not port_obj:
@@ -562 +568 @@
             options, parsed_args = run_webkit_tests.parse_args(args)
             test_port = ImageDiffTestPort(options=options, user=mocktool.MockUser())
-            passing_run(args, port_obj=test_port, tests_included=True)
+            res = passing_run(args, port_obj=test_port, tests_included=True)
+            self.assertTrue(res)
             return test_port
 
-        base_args = ['--pixel-tests', 'failures/expected/*']
+        base_args = ['--pixel-tests', '--no-new-test-results', 'failures/expected/*']
 
         # If we pass in an explicit tolerance argument, then that will be used.
@@ -642 +649 @@
 
 class RebaselineTest(unittest.TestCase):
-    def assertBaselines(self, file_list, file, extensions):
+    def assertBaselines(self, file_list, file, extensions, err):
         "assert that the file_list contains the baselines."""
         for ext in extensions:
             baseline = file + "-expected" + ext
+            baseline_msg = 'Writing new expected result "%s"\n' % baseline[1:]
             self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
+            self.assertTrue(baseline_msg in err.get())
 
     # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
@@ -655 +664 @@
         # is missing, update the expected generic location.
         fs = port.unit_test_filesystem()
-        passing_run(['--pixel-tests',
+        res, out, err, _ = logging_run(['--pixel-tests',
                         '--reset-results',
                         'passes/image.html',
                         'failures/expected/missing_image.html'],
-                        tests_included=True, filesystem=fs)
+                        tests_included=True, filesystem=fs, new_results=True)
         file_list = fs.written_files.keys()
         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
+        self.assertEquals(res, 0)
+        self.assertTrue(out.empty())
         self.assertEqual(len(file_list), 4)
-        self.assertBaselines(file_list, "/passes/image", [".txt", ".png"])
-        self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"])
+        self.assertBaselines(file_list, "/passes/image", [".txt", ".png"], err)
+        self.assertBaselines(file_list, "/failures/expected/missing_image", [".txt", ".png"], err)
 
     def test_missing_results(self):
@@ -670 +681 @@
         # is missing, update the expected generic location.
         fs = port.unit_test_filesystem()
-        passing_run(['--no-show-results',
+        res, out, err, _ = logging_run(['--no-show-results',
                      'failures/unexpected/missing_text.html',
                      'failures/unexpected/missing_image.html',
                      'failures/unexpected/missing_audio.html'],
-                     tests_included=True, filesystem=fs)
+                     tests_included=True, filesystem=fs, new_results=True)
         file_list = fs.written_files.keys()
         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
+        self.assertEquals(res, 2)
+        self.assertFalse(out.empty())
         self.assertEqual(len(file_list), 4)
-        self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"])
-        self.assertBaselines(file_list, "/failures/unexpected/missing_image", [".png"])
+        self.assertBaselines(file_list, "/failures/unexpected/missing_text", [".txt"], err)
+        self.assertBaselines(file_list, "/failures/unexpected/missing_image", [".png"], err)
 
     def test_new_baseline(self):
@@ -685 +698 @@
         # is mssing, then create a new expectation in the platform dir.
         fs = port.unit_test_filesystem()
-        passing_run(['--pixel-tests',
+        res, out, err, _ = logging_run(['--pixel-tests',
                         '--new-baseline',
                         'passes/image.html',
                         'failures/expected/missing_image.html'],
-                    tests_included=True, filesystem=fs)
+                    tests_included=True, filesystem=fs, new_results=True)
         file_list = fs.written_files.keys()
         file_list.remove('/tmp/layout-test-results/tests_run0.txt')
+        self.assertEquals(res, 0)
+        self.assertTrue(out.empty())
         self.assertEqual(len(file_list), 4)
         self.assertBaselines(file_list,
-            "/platform/test-mac-leopard/passes/image", [".txt", ".png"])
+            "/platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
         self.assertBaselines(file_list,
-            "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"])
+            "/platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)