Changeset 247781 in webkit

Timestamp:

Jul 24, 2019 11:07:01 AM (5 years ago)

Author:

commit-queue@webkit.org

Message:

run-javascriptcore-tests won't report test results for testmasm, testair, testb3, testdfg and test api
​https://bugs.webkit.org/show_bug.cgi?id=199489

Patch by Zhifei Fang <​zhifei_fang@apple.com> on 2019-07-24
Reviewed by Aakash Jain.

• BuildSlaveSupport/build.webkit.org-config/steps.py:

(RunJavaScriptCoreTests.start): Skip run jsc test binaries for mips, armv7, aarch64.
(RunJavaScriptCoreTests.countFailures): Pick up the test binaires failure to total failures count

• BuildSlaveSupport/build.webkit.org-config/steps_unittest.py:
• Scripts/run-javascriptcore-tests: The script now failed wether the run-jsc-stress-tests failed or test binaries failed.

(runTest): Record test binary failure
(reportTestFailures): Report test binary failure
(runJSCStressTests):

• Scripts/webkitpy/common/config/ports.py: Skip run jsc test binaries for those need to run remotely

(JscOnlyPort.build_jsc_command):
(JscOnlyPort):
(JscOnlyPort.run_javascriptcore_tests_command):

• Scripts/webkitpy/common/config/ports_unittest.py:

(DeprecatedPortTest.test_jsconly_port):

Location:

trunk/Tools

Files:

• BuildSlaveSupport/build.webkit.org-config/steps.py (modified) (2 diffs)
• BuildSlaveSupport/build.webkit.org-config/steps_unittest.py (modified) (1 diff)
• ChangeLog (modified) (1 diff)
• Scripts/run-javascriptcore-tests (modified) (8 diffs)
• Scripts/webkitpy/common/config/ports.py (modified) (1 diff)
• Scripts/webkitpy/common/config/ports_unittest.py (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps.py

@@ -332 +332 @@
     def start(self):
         platform = self.getProperty('platform')
+        architecture = self.getProperty("architecture")
+        # Currently run-javascriptcore-test doesn't support run javascript core test binaries list below remotely
+        if architecture in ['mips', 'armv7', 'aarch64']:
+            self.command += ['--no-testmasm', '--no-testair', '--no-testb3', '--no-testdfg', '--no-testapi']
         # Linux bots have currently problems with JSC tests that try to use large amounts of memory.
         # Check: https://bugs.webkit.org/show_bug.cgi?id=175140
@@ -345 +349 @@
     def countFailures(self, cmd):
         logText = cmd.logs['stdio'].getText()
+        count = 0
 
         match = re.search(r'^Results for JSC stress tests:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
         if match:
-            return int(match.group(1))
+            count += int(match.group(1))
+
+        match = re.search(r'Results for JSC test binaries:\r?\n\s+(\d+) failure', logText, re.MULTILINE)
+        if match:
+            count += int(match.group(1))
 
         match = re.search(r'^Results for Mozilla tests:\r?\n\s+(\d+) regression', logText, re.MULTILINE)
         if match:
-            return int(match.group(1))
-
-        return 0
+            count += int(match.group(1))
+
+        return count
 
 

trunk/Tools/BuildSlaveSupport/build.webkit.org-config/steps_unittest.py

@@ -123 +123 @@
     5 failures found.""")
 
+    def test_jsc_stress_failures_with_binary_results_output(self):
+        self.assertResults(FAILURE, ["8 JSC tests failed"], 1,  """Results for JSC stress tests:
+    5 failures found.
+Results for JSC test binaries:
+    3 failures found.""")
+
+    def test_jsc_stress_failures_with_binary_result_output(self):
+        self.assertResults(FAILURE, ["6 JSC tests failed"], 1,  """Results for JSC stress tests:
+    5 failures found.
+Results for JSC test binaries:
+    1 failure found.""")
+
 
 class RunTest262TestsTest(unittest.TestCase):

trunk/Tools/ChangeLog

@@ +1 @@
+2019-07-24  Zhifei Fang  <zhifei_fang@apple.com>
+
+        run-javascriptcore-tests won't report test results for testmasm, testair, testb3, testdfg and test api
+        https://bugs.webkit.org/show_bug.cgi?id=199489
+
+        Reviewed by Aakash Jain.
+
+        * BuildSlaveSupport/build.webkit.org-config/steps.py:
+        (RunJavaScriptCoreTests.start): Skip run jsc test binaries for mips, armv7, aarch64.
+        (RunJavaScriptCoreTests.countFailures): Pick up the test binaires failure to total failures count
+        * BuildSlaveSupport/build.webkit.org-config/steps_unittest.py:
+        * Scripts/run-javascriptcore-tests: The script now failed wether the run-jsc-stress-tests failed or test binaries failed.
+        (runTest): Record test binary failure
+        (reportTestFailures): Report test binary failure
+        (runJSCStressTests): 
+        * Scripts/webkitpy/common/config/ports.py: Skip run jsc test binaries for those need to run remotely
+        (JscOnlyPort.build_jsc_command):
+        (JscOnlyPort):
+        (JscOnlyPort.run_javascriptcore_tests_command):
+        * Scripts/webkitpy/common/config/ports_unittest.py:
+        (DeprecatedPortTest.test_jsconly_port):
+
 2019-07-24  Zalan Bujtas  <zalan@apple.com>
 

trunk/Tools/Scripts/run-javascriptcore-tests

@@ -93 +93 @@
 my $failFast = 1;
 my %jsonData = ();
+my @testResults = ();
+my $isTestFailed = 0;
 my $remoteConfigFile;
 my $jsonFileName;
@@ -234 +236 @@
   --make-runner                 Uses the faster make-based runner.
   --ruby-runner                 Uses the ruby runner for machines without unix shell or make.
- 
+
   --test-writer [writer]        Specifies the test script format."
                                 default is to use shell scripts to run the tests"
@@ -399 +401 @@
     print "$testName completed with rc=$testResult ($exitStatus)\n\n";
 
+    if ($testResult) {
+        $isTestFailed = 1;
+        push @testResults, $testName;
+    }
     if (defined($jsonFileName)) {
         my $testStatus = ($exitStatus == 0)? JSON::PP::true: JSON::PP::false;
@@ -405 +411 @@
 
     if ($testResult && $failFast) {
+        reportTestFailures();
         writeJsonDataIfApplicable();
         exit exitStatus($testResult);
     }
+}
+
+sub reportTestFailures {
+  my $numJSCtestFailures = @testResults;
+  if ($numJSCtestFailures) {
+      print "\n** The following JSC test binaries failures have been introduced:\n";
+      foreach my $testFailure (@testResults) {
+          print "\t$testFailure\n";
+      }
+  }
+  print "\n";
+
+  print "Results for JSC test binaries:\n";
+  printThingsFound($numJSCtestFailures, "failure", "failures", "found");
+  print "    OK.\n" if $numJSCtestFailures == 0;
+
+  print "\n";
 }
 
@@ -421 +445 @@
 
 runJSCStressTests();
+reportTestFailures();
+
+if ($isTestFailed) {
+  exit(1);
+}
 
 sub runJSCStressTests
@@ -474 +503 @@
         push(@jscStressDriverCmd, "--debug");
     }
-    
+
     if (!$copyJSC) {
         push(@jscStressDriverCmd, "--no-copy");
@@ -574 +603 @@
 
     if ($numJSCStressFailures) {
+        $isTestFailed = 1;
         print "\n** The following JSC stress test failures have been introduced:\n";
         foreach my $testFailure (@jscStressFailList) {
@@ -592 +622 @@
 
     writeJsonDataIfApplicable();
-    exit(1) if $numJSCStressFailures;
 }
 

trunk/Tools/Scripts/webkitpy/common/config/ports.py

@@ -244 +244 @@
         command.append("--jsc-only")
         return self._append_build_style_flag(command, build_style)
+
+    def run_javascriptcore_tests_command(self, build_style=None):
+        command = self.script_shell_command("run-javascriptcore-tests")
+        command.append("--no-fail-fast")
+        command.append("--no-testmasm")
+        command.append("--no-testair")
+        command.append("--no-testb3")
+        command.append("--no-testdfg")
+        command.append("--no-testapi")
+        if 'JSCTESTS_OPTIONS' in os.environ:
+            command += os.environ['JSCTESTS_OPTIONS'].split()
+        return self._append_build_style_flag(command, build_style)

trunk/Tools/Scripts/webkitpy/common/config/ports_unittest.py

@@ -68 +68 @@
         self.assertEqual(JscOnlyPort().build_jsc_command(build_style="release"), DeprecatedPort().script_shell_command("build-jsc") + ["--jsc-only", "--release"])
         self.assertEqual(JscOnlyPort().build_jsc_command(build_style="debug"), DeprecatedPort().script_shell_command("build-jsc") + ["--jsc-only", "--debug"])
+        self.assertEqual(JscOnlyPort().run_javascriptcore_tests_command(build_style="debug"), DeprecatedPort().script_shell_command("run-javascriptcore-tests") + ['--no-fail-fast', '--no-testmasm', '--no-testair', '--no-testb3', '--no-testdfg', '--no-testapi', '--debug'])