Changeset 232841 in webkit

Timestamp:

Jun 14, 2018 10:49:01 AM (6 years ago)

Author:

commit-queue@webkit.org

Message:

[test262-runner] Test output should summarize tests that are
unexpectedly passing/failing.
​https://bugs.webkit.org/show_bug.cgi?id=186527

Patch by Valerie R Young <​valerie@bocoup.com> on 2018-06-14
Reviewed by Michael Saboff.

• Scripts/test262/Runner.pm:

(main):

In verbose mode, a summary of all new failing tests
and all new passing tests are printed at the end of
the script output.

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/test262/Runner.pm (modified) (2 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2018-06-14  Valerie R Young  <valerie@bocoup.com>
+
+        [test262-runner] Test output should summarize tests that are
+        unexpectedly passing/failing.
+        https://bugs.webkit.org/show_bug.cgi?id=186527
+
+        Reviewed by Michael Saboff.
+
+        * Scripts/test262/Runner.pm:
+        (main):
+          In verbose mode, a summary of all new failing tests
+          and all new passing tests are printed at the end of
+          the script output.
+
 2018-06-14  Zan Dobersek  <zdobersek@igalia.com>
 

trunk/Tools/Scripts/test262/Runner.pm

@@ -359 +359 @@
     my $newpasscount = 0;
     my $skipfilecount = 0;
+    my $newfailurereport = '';
+    my $newpassreport = '';
 
     # Create expectation file and calculate results
@@ -382 +384 @@
 
             # If an unexpected failure
-            $newfailcount++ if !$expectedFailure || ($expectedFailure ne $test->{error});
+            if (!$expectedFailure || ($expectedFailure ne $test->{error})) {
+                $newfailcount++;
+
+                if ($verbose) {
+                    my $path = $test->{path};
+                    my $mode = $test->{mode};
+                    my $err = $test->{error};
+                    $newfailurereport .= "FAIL $path ($mode)\n$err\n\n";
+                }
+            }
 
         }
         elsif ($test->{result} eq 'PASS') {
             # If this is an newly passing test
-            $newpasscount++ if $expectedFailure;
+            if ($expectedFailure) {
+                $newpasscount++;
+
+                if ($verbose) {
+                    my $path = $test->{path};
+                    my $mode = $test->{mode};
+                    $newpassreport .= "PASS $path ($mode)\n";
+                }
+            }
         }
         elsif ($test->{result} eq 'SKIP') {
             $skipfilecount++;
         }
+    }
+
+    # In verbose mode, the result of every test is printed, so summarize useful results
+    if ($verbose && $expect && ($newfailurereport || $newpassreport)) {
+        print "\n";
+        if ($newfailurereport) {
+            print "---------------NEW FAILING TESTS SUMMARY---------------\n\n";
+            print "$newfailurereport";
+        }
+        if ($newpassreport) {
+            print "---------------NEW PASSING TESTS SUMMARY---------------\n\n";
+            print "$newpassreport\n";
+        }
+        print "---------------------------------------------------------\n";
     }